Merge pull request #178 from runpod-workers/up-0.8.4

update vllm
2025-04-21 13:15:03 -07:00 · 2025-04-21 13:14:32 -07:00 · 2025-04-21 11:39:53 -07:00 · 2025-04-07 11:58:03 -07:00 · 2025-04-07 11:54:57 -07:00 · 2025-04-03 14:52:06 -07:00
8 changed files with 1194 additions and 36 deletions
@@ -0,0 +1,44 @@
+{
+    "tests": [
+        {
+            "name": "basic_inference_test",
+            "input": {
+                "prompt": "Write a short poem about artificial intelligence."
+            },
+            "timeout": 30000
+        }
+    ],
+    "config": {
+        "gpuTypeId": "NVIDIA GeForce RTX 4090",
+        "gpuCount": 1,
+        "env": [
+            {
+                "key": "MODEL_NAME",
+                "value": "facebook/opt-350m"
+            },
+            {
+                "key": "HF_TOKEN",
+                "value": "hf_dummy_token_for_testing_purposes_only"
+            },
+            {
+                "key": "MAX_MODEL_LEN",
+                "value": "8192"
+            },
+            {
+                "key": "GPU_MEMORY_UTILIZATION",
+                "value": "0.95"
+            }
+        ],
+        "allowedCudaVersions": [
+            "12.7",
+            "12.6",
+            "12.5",
+            "12.4",
+            "12.3",
+            "12.2",
+            "12.1",
+            "12.0",
+            "11.7"
+        ]
+    }
+}
@@ -12,7 +12,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
    python3 -m pip install --upgrade -r /requirements.txt

 # Install vLLM (switching back to pip installs since issues that required building fork are fixed and space optimization is not as important since caching) and FlashInfer 
-RUN python3 -m pip install vllm==0.8.2 && \
+RUN python3 -m pip install vllm==0.8.4 && \
    python3 -m pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3

 # Setup for Option 2: Building the Image with the Model included
@@ -18,9 +18,9 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https:
 ### 1. UI for Deploying vLLM Worker on RunPod console:
 ![Demo of Deploying vLLM Worker on RunPod console with new UI](media/ui_demo.gif)

-### 2. Worker vLLM `v2.2.0` with vLLM `0.8.2` now available under `stable` tags 
+### 2. Worker vLLM `v2.4.0` with vLLM `0.8.4` now available under `stable` tags 

-Update v2.2.0 is now available, use the image tag `runpod/worker-v1-vllm:v2.2.0stable-cuda12.1.0`.
+Update v2.3.0 is now available, use the image tag `runpod/worker-v1-vllm:v2.4.0stable-cuda12.1.0`.

 ### 3. OpenAI-Compatible [Embedding Worker](https://github.com/runpod-workers/worker-infinity-embedding) Released
 Deploy your own OpenAI-compatible Serverless Endpoint on RunPod with multiple embedding models and fast inference for RAG and more! 
@@ -82,7 +82,7 @@ Below is a summary of the available RunPod Worker images, categorized by image s

 | CUDA Version | Stable Image Tag                  | Development Image Tag             | Note                                                        |
 |--------------|-----------------------------------|-----------------------------------|----------------------------------------------------------------------|
-| 12.1.0       | `runpod/worker-v1-vllm:v2.2.0stable-cuda12.1.0` | `runpod/worker-v1-vllm:v2.2.0dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. |
+| 12.1.0       | `runpod/worker-v1-vllm:v2.4.0stable-cuda12.1.0` | `runpod/worker-v1-vllm:v2.4.0dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. |



@@ -125,7 +125,7 @@ Below is a summary of the available RunPod Worker images, categorized by image s
 | `MAX_NUM_SEQS`                            | 256                   | `int`                                      | Maximum number of sequences per iteration. |
 | `MAX_LOGPROBS`                            | 20                    | `int`                                      | Max number of log probs to return when logprobs is specified in SamplingParams. |
 | `DISABLE_LOG_STATS`                       | False                 | `bool`                                     | Disable logging statistics. |
-| `QUANTIZATION`                            | None                  | ['awq', 'squeezellm', 'gptq']              | Method used to quantize the weights. |
+| `QUANTIZATION`                            | None                  | ['awq', 'squeezellm', 'gptq', 'bitsandbytes']              | Method used to quantize the weights. |
 | `ROPE_SCALING`                            | None                  | `dict`                                     | RoPE scaling configuration in JSON format. |
 | `ROPE_THETA`                              | None                  | `float`                                    | RoPE theta. Use with rope_scaling. |
 | `TOKENIZER_POOL_SIZE`                     | 0                     | `int`                                      | Size of tokenizer pool to use for asynchronous tokenization. |
@@ -4,8 +4,9 @@ pyarrow
 runpod~=1.7.7
 huggingface-hub
 packaging
-typing-extensions==4.7.1
+typing-extensions>=4.8.0
 pydantic
 pydantic-settings
 hf-transfer
 transformers
+bitsandbytes>=0.45.0
@@ -7,7 +7,7 @@ variable "REPOSITORY" {
 }

 variable "BASE_IMAGE_VERSION" {
-  default = "v2.0.0stable"
+  default = "v2.4.0stable"
 }

 group "all" {
@@ -148,6 +148,9 @@ def get_engine_args():
    # Rename and match to vllm args
    args = match_vllm_args(args)

+    if args.get("load_format") == "bitsandbytes":
+        args["quantization"] = args["load_format"]
+    
    # Set tensor parallel size and max parallel loading workers if more than 1 GPU is available
    num_gpus = device_count()
    if num_gpus > 1:
@@ -1,5 +1,127 @@
 {
  "versions": {
+    "0.8.4": {
+      "imageName": "runpod/worker-v1-vllm:v2.4.0stable-cuda12.1.0",
+      "minimumCudaVersion": "12.1",
+      "categories": [
+        {
+          "title": "LLM Settings",
+          "settings": [
+            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
+            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
+            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
+            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
+            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
+            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
+            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
+            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
+            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
+            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
+            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
+            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
+            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
+            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
+            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
+            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
+            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
+            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
+            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
+          ]
+        },
+        {
+          "title": "Tokenizer Settings",
+          "settings": [
+            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
+          ]
+        },
+        {
+          "title": "System Settings",
+          "settings": [
+            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
+            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
+          ]
+        },
+        {
+          "title": "Streaming Settings",
+          "settings": [
+            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
+          ]
+        },
+        {
+          "title": "OpenAI Settings",
+          "settings": [
+            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
+          ]
+        },
+        {
+          "title": "Serverless Settings",
+          "settings": [
+            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
+          ]
+        }
+      ]
+    },
+    "0.8.3": {
+      "imageName": "runpod/worker-v1-vllm:v2.3.0stable-cuda12.1.0",
+      "minimumCudaVersion": "12.1",
+      "categories": [
+        {
+          "title": "LLM Settings",
+          "settings": [
+            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
+            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
+            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
+            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
+            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
+            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
+            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
+            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
+            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
+            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
+            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
+            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
+            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
+            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
+            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
+            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
+            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
+            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
+            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
+          ]
+        },
+        {
+          "title": "Tokenizer Settings",
+          "settings": [
+            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
+          ]
+        },
+        {
+          "title": "System Settings",
+          "settings": [
+            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
+            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
+          ]
+        },
+        {
+          "title": "Streaming Settings",
+          "settings": [
+            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
+          ]
+        },
+        {
+          "title": "OpenAI Settings",
+          "settings": [
+            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
+          ]
+        },
+        {
+          "title": "Serverless Settings",
+          "settings": [
+            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
+          ]
+        }
+      ]
+    },
    "0.8.2": {
      "imageName": "runpod/worker-v1-vllm:v2.2.0stable-cuda12.1.0",
      "minimumCudaVersion": "12.1",
@@ -741,14 +863,15 @@
    "env_var_name": "QUANTIZATION",
    "value": "",
    "title": "Quantization",
-    "description": "Method used to quantize the weights.",
+    "description": "Method used to quantize the weights.\nif the `Load Format` is 'bitsandbytes' then `Quantization` will be forced to 'bitsandbytes'",
    "required": false,
    "type": "select",
    "options": [
      { "value": "None", "label": "None" },
      { "value": "awq", "label": "AWQ" },
      { "value": "squeezellm", "label": "SqueezeLLM" },
-      { "value": "gptq", "label": "GPTQ" }
+      { "value": "gptq", "label": "GPTQ" },
+      { "value": "bitsandbytes", "label": "bitsandbytes" }
    ]
  },
  "ROPE_SCALING": {
Author	SHA1	Message	Date
Marut PandyaandGitHub	e807342e90	Merge pull request #178 from runpod-workers/up-0.8.4 update vllm	2025-04-21 13:15:03 -07:00
pandyamarut	f33e8d2bcd	update vllm Signed-off-by: pandyamarut <pandyamarut@gmail.com>	2025-04-21 13:14:32 -07:00
Marut PandyaandGitHub	6b64bb93bd	Merge pull request #147 from mohamednaji7/BitsAndBytes completing the "bitsandbytes" option - based on https://docs.vllm.ai/en/stable/quantization/bnb.html	2025-04-21 11:39:53 -07:00
Marut PandyaandGitHub	d77c53c3b7	Merge pull request #175 from runpod-workers/up-0.8.3 update vllm	2025-04-07 11:58:03 -07:00
pandyamarut	3d067cd472	update vllm Signed-off-by: pandyamarut <pandyamarut@gmail.com>	2025-04-07 11:54:57 -07:00
Marut PandyaandGitHub	f1360ccae7	Merge pull request #173 from KAJdev/patch-1 Update hub.json	2025-04-03 14:52:06 -07:00
Marut PandyaandGitHub	96f1b86126	Merge pull request #174 from KAJdev/patch-2 Update tests.json	2025-04-03 10:14:49 -07:00
Ezekiel WotringandGitHub	3a095f3e10	Update tests.json	2025-04-03 09:10:01 -08:00
Ezekiel WotringandGitHub	7cf7f3e4e6	Update hub.json	2025-04-03 09:04:28 -08:00
Marut PandyaandGitHub	4395d0c67b	Update hub.json	2025-04-03 08:10:28 -07:00
Marut PandyaandGitHub	5864fa6843	Update hub.json	2025-03-31 15:55:35 -07:00
Marut PandyaandGitHub	067f0bc173	Update tests.json	2025-03-31 15:49:31 -07:00
Marut PandyaandGitHub	0869068e7c	Create tests.json	2025-03-31 15:32:06 -07:00
Marut PandyaandGitHub	19f25b17de	Update hub.json	2025-03-31 15:14:40 -07:00
Marut PandyaandGitHub	70aa748c30	Update hub.json	2025-03-31 15:14:24 -07:00
Mohamed NagyandGitHub	04288240f6	updating the `.` in ['awq', 'squeezellm', 'gptq'. 'bitsandbytes'] for the QUNATIZATION row	2025-02-02 14:58:48 +02:00
Mohamed NagyandGitHub	9299f43b8b	solving "typing_extensions" compatibility with "bitsandbytes" ``` 2025-01-22 18:04:01 [INFO] > [stage-0 5/8] RUN --mount=type=cache,target=/root/.cache/pip python3 -m pip install --upgrade pip && python3 -m pip install --upgrade -r /requirements.txt: 2025-01-22 18:04:01 [INFO] #13 8.904 2025-01-22 18:04:01 [INFO] #13 8.904 The conflict is caused by: 2025-01-22 18:04:01 [INFO] #13 8.904 The user requested typing-extensions==4.7.1 2025-01-22 18:04:01 [INFO] #13 8.904 bitsandbytes 0.45.0 depends on typing_extensions>=4.8.0 ```	2025-01-22 18:06:46 +02:00
mohamednaji7	131c17569f	correct access to "args" dictionary	2025-01-21 22:33:28 +02:00
mohamednaji7	8882f6d50b	adding 'bitsandbytes' to `QUANTIZATION`	2025-01-21 14:46:40 +02:00
mohamednaji7	331bc30101	adding 'bitsandbytes' option	2025-01-21 14:08:37 +02:00
mohamednaji7	a27f72a33a	inforce args.quantization for bnb load_froamt	2025-01-21 14:01:32 +02:00
mohamednaji7	7167985f23	including bitsandbytes "src:https://docs.vllm.ai/en/stable/quantization/bnb.html "	2025-01-21 13:49:36 +02:00