Update README.md

[HF]: set default max_token size

.github/workflows/CI-runpod_dep.yml

@@ -19,32 +19,49 @@ jobs:
 
       - name: Check for new package version and update
         run: |
-          # Get current version
-          current_version=$(grep -oP 'runpod==\K[^"]+' ./builder/requirements.txt)
+          echo "Fetching the current runpod version from requirements.txt..."
+          
+          # Get current version, allowing both == and ~= in the search pattern
+          current_version=$(grep -oP 'runpod[~=]{1,2}\K[^"]+' ./builder/requirements.txt)
+          echo "Current version: $current_version"
 
-          # Get new version
+          # Extract major and minor from current version
+          current_major_minor=$(echo $current_version | cut -d. -f1,2)
+          echo "Current major.minor: $current_major_minor"
+
+          echo "Fetching the latest runpod version from PyPI..."
+          
+          # Get new version from PyPI
           new_version=$(curl -s https://pypi.org/pypi/runpod/json | jq -r .info.version)
           echo "NEW_VERSION_ENV=$new_version" >> $GITHUB_ENV
+          echo "New version: $new_version"
+
+          # Extract major and minor from new version
+          new_major_minor=$(echo $new_version | cut -d. -f1,2)
+          echo "New major.minor: $new_major_minor"
 
           if [ -z "$new_version" ]; then
-              echo "Failed to fetch the new version."
+              echo "ERROR: Failed to fetch the new version from PyPI."
               exit 1
           fi
 
-          # Check if the version is already up-to-date
-          if [ "$current_version" = "$new_version" ]; then
-              echo "The package version is already up-to-date."
+          # Check if the major or minor version is different
+          if [ "$current_major_minor" = "$new_major_minor" ]; then
+              echo "No update needed. The new version ($new_major_minor) is within the allowed range (~= $current_major_minor)."
               exit 0
           fi
 
-          # Update requirements.txt
-          sed -i "s/runpod==.*/runpod==$new_version/" ./builder/requirements.txt
+          echo "New major/minor detected ($new_major_minor). Updating requirements.txt..."
+
+          # Update requirements.txt, preserving the existing constraint type (~= or ==)
+          sed -i "s/runpod[~=][^ ]*/runpod~=$new_version/" ./builder/requirements.txt
+          echo "requirements.txt has been updated."
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v3
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: Update package version
+          commit-message: Update runpod package version
           title: Update runpod package version
           body: The package version has been updated to ${{ env.NEW_VERSION_ENV }}
           branch: runpod-package-update

Dockerfile

@@ -12,7 +12,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     python3 -m pip install --upgrade -r /requirements.txt
 
 # Install vLLM (switching back to pip installs since issues that required building fork are fixed and space optimization is not as important since caching) and FlashInfer 
-RUN python3 -m pip install vllm==0.5.4 && \
+RUN python3 -m pip install vllm==0.7.3 && \
     python3 -m pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3
 
 # Setup for Option 2: Building the Image with the Model included
@@ -32,7 +32,7 @@ ENV MODEL_NAME=$MODEL_NAME \
     HF_DATASETS_CACHE="${BASE_PATH}/huggingface-cache/datasets" \
     HUGGINGFACE_HUB_CACHE="${BASE_PATH}/huggingface-cache/hub" \
     HF_HOME="${BASE_PATH}/huggingface-cache/hub" \
-    HF_HUB_ENABLE_HF_TRANSFER=1 
+    HF_HUB_ENABLE_HF_TRANSFER=0 
 
 ENV PYTHONPATH="/:/vllm-workspace"
 
@@ -40,10 +40,10 @@ ENV PYTHONPATH="/:/vllm-workspace"
 COPY src /src
 RUN --mount=type=secret,id=HF_TOKEN,required=false \
     if [ -f /run/secrets/HF_TOKEN ]; then \
-        export HF_TOKEN=$(cat /run/secrets/HF_TOKEN); \
+    export HF_TOKEN=$(cat /run/secrets/HF_TOKEN); \
     fi && \
     if [ -n "$MODEL_NAME" ]; then \
-        python3 /src/download_model.py; \
+    python3 /src/download_model.py; \
     fi
 
 # Start the handler

README.md

@@ -18,9 +18,9 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https:
 ### 1. UI for Deploying vLLM Worker on RunPod console:
 ![Demo of Deploying vLLM Worker on RunPod console with new UI](media/ui_demo.gif)
 
-### 2. Worker vLLM `v1.2.0` with vLLM `0.5.4` now available under `stable` tags 
+### 2. Worker vLLM `v2.1.0` with vLLM `0.7.3` now available under `stable` tags 
 
-Update v1.2.0 is now available, use the image tag `runpod/worker-v1-vllm:v1.2.0stable-cuda12.1.0`.
+Update v2.0.0 is now available, use the image tag `runpod/worker-v1-vllm:v2.1.0stable-cuda12.1.0`.
 
 ### 3. OpenAI-Compatible [Embedding Worker](https://github.com/runpod-workers/worker-infinity-embedding) Released
 Deploy your own OpenAI-compatible Serverless Endpoint on RunPod with multiple embedding models and fast inference for RAG and more! 
@@ -82,7 +82,7 @@ Below is a summary of the available RunPod Worker images, categorized by image s
 
 | CUDA Version | Stable Image Tag                  | Development Image Tag             | Note                                                        |
 |--------------|-----------------------------------|-----------------------------------|----------------------------------------------------------------------|
-| 12.1.0       | `runpod/worker-v1-vllm:stable-cuda12.1.0` | `runpod/worker-v1-vllm:dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. |
+| 12.1.0       | `runpod/worker-v1-vllm:v2.1.0stable-cuda12.1.0` | `runpod/worker-v1-vllm:v2.1.0dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. |
 
 
 
@@ -600,4 +600,4 @@ The JSON consists of two main parts, schema and versions.
     }
   }
 }
-```
+```

builder/requirements.txt

@@ -1,10 +1,11 @@
 ray
 pandas
 pyarrow
-runpod==1.7.0
+runpod~=1.7.0
 huggingface-hub
 packaging
 typing-extensions==4.7.1
 pydantic
 pydantic-settings
-hf-transfer
+hf-transfer
+transformers

docker-bake.hcl

@@ -7,7 +7,7 @@ variable "REPOSITORY" {
 }
 
 variable "BASE_IMAGE_VERSION" {
-  default = "stable"
+  default = "v2.0.0stable"
 }
 
 group "all" {

src/engine.py

@@ -4,13 +4,16 @@ import json
 import asyncio
 
 from dotenv import load_dotenv
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Optional
 import time
 
 from vllm import AsyncLLMEngine
+from vllm.entrypoints.logger import RequestLogger
 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
 from vllm.entrypoints.openai.protocol import ChatCompletionRequest, CompletionRequest, ErrorResponse
+from vllm.entrypoints.openai.serving_models import BaseModelPath, LoRAModulePath, OpenAIServingModels
+
 
 from utils import DummyRequest, JobInput, BatchSize, create_error_response
 from constants import DEFAULT_MAX_CONCURRENCY, DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE_GROWTH_FACTOR, DEFAULT_MIN_BATCH_SIZE
@@ -121,27 +124,50 @@ class OpenAIvLLMEngine(vLLMEngine):
         self.response_role = os.getenv("OPENAI_RESPONSE_ROLE") or "assistant"
         asyncio.run(self._initialize_engines())
         self.raw_openai_output = bool(int(os.getenv("RAW_OPENAI_OUTPUT", 1)))
-
+        
     async def _initialize_engines(self):
         self.model_config = await self.llm.get_model_config()
-        
-        self.chat_engine = OpenAIServingChat(
-            async_engine_client=self.llm, 
+        self.base_model_paths = [
+            BaseModelPath(name=self.engine_args.model, model_path=self.engine_args.model)
+        ]
+
+        lora_modules = os.getenv('LORA_MODULES', None)
+        if lora_modules is not None:
+            try:
+                lora_modules = json.loads(lora_modules)
+                lora_modules = [LoRAModulePath(**lora_modules)]
+            except:
+                lora_modules = None
+
+        self.serving_models = OpenAIServingModels(
+            engine_client=self.llm,
             model_config=self.model_config,
-            served_model_names=[self.served_model_name], 
-            response_role=self.response_role,
-            chat_template=self.tokenizer.tokenizer.chat_template,
+            base_model_paths=self.base_model_paths,
             lora_modules=None,
             prompt_adapters=None,
-            request_logger=None
+        )
+
+        self.chat_engine = OpenAIServingChat(
+            engine_client=self.llm, 
+            model_config=self.model_config,
+            models=self.serving_models,
+            response_role=self.response_role,
+            request_logger=None,
+            chat_template=self.tokenizer.tokenizer.chat_template,
+            chat_template_content_format="auto",
+            # enable_reasoning=os.getenv('ENABLE_REASONING', 'false').lower() == 'true',
+            # reasoning_parser=None,
+            # return_token_as_token_ids=False,
+            enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
+            tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None,
+            enable_prompt_tokens_details=False
         )
         self.completion_engine = OpenAIServingCompletion(
-            async_engine_client=self.llm, 
+            engine_client=self.llm, 
             model_config=self.model_config,
-            served_model_names=[self.served_model_name],
-            lora_modules=[],
-            prompt_adapters=None,
-            request_logger=None
+            models=self.serving_models,
+            request_logger=None,
+            # return_token_as_token_ids=False,
         )
     
     async def generate(self, openai_request: JobInput):
@@ -154,10 +180,7 @@ class OpenAIvLLMEngine(vLLMEngine):
             yield create_error_response("Invalid route").model_dump()
     
     async def _handle_model_request(self):
-        models = await self.chat_engine.show_available_models()
-        fixed_model = models.data[0]
-        fixed_model.id = self.served_model_name
-        models.data = [fixed_model]
+        models = await self.serving_models.show_available_models()
         return models.model_dump()
     
     async def _handle_chat_or_completion_request(self, openai_request: JobInput):
@@ -176,7 +199,8 @@ class OpenAIvLLMEngine(vLLMEngine):
             yield create_error_response(str(e)).model_dump()
             return
         
-        response_generator = await generator_function(request, raw_request=None)
+        dummy_request = DummyRequest()
+        response_generator = await generator_function(request, raw_request=dummy_request)
 
         if not openai_request.openai_input.get("stream") or isinstance(response_generator, ErrorResponse):
             yield response_generator.model_dump()

src/engine_args.py

@@ -4,6 +4,7 @@ import logging
 from torch.cuda import device_count
 from vllm import AsyncEngineArgs
 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
+from src.utils import convert_limit_mm_per_prompt
 
 RENAME_ARGS_MAP = {
     "MODEL_NAME": "model",
@@ -88,7 +89,8 @@ DEFAULT_ARGS = {
     "typical_acceptance_sampler_posterior_alpha": float(os.getenv('TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA', 0)) or None,
     "qlora_adapter_name_or_path": os.getenv('QLORA_ADAPTER_NAME_OR_PATH', None),
     "disable_logprobs_during_spec_decoding": os.getenv('DISABLE_LOGPROBS_DURING_SPEC_DECODING', None),
-    "otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None)
+    "otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None),
+    "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true'),
 }
 
 def match_vllm_args(args):

src/utils.py

@@ -3,6 +3,7 @@ import logging
 from http import HTTPStatus
 from functools import wraps
 from time import time
+from vllm.entrypoints.openai.protocol import RequestResponseMetadata
 
 try:
     from vllm.utils import random_uuid
@@ -14,6 +15,10 @@ except ImportError:
 
 logging.basicConfig(level=logging.INFO)
 
+def convert_limit_mm_per_prompt(input_string: str):
+    key, value = input_string.split('=')
+    return {key: int(value)}
+
 def count_physical_cores():
     with open('/proc/cpuinfo') as f:
         content = f.readlines()
@@ -39,7 +44,11 @@ class JobInput:
         self.max_batch_size = job.get("max_batch_size")
         self.apply_chat_template = job.get("apply_chat_template", False)
         self.use_openai_format = job.get("use_openai_format", False)
-        self.sampling_params = SamplingParams(**job.get("sampling_params", {}))
+        samp_param = job.get("sampling_params", {})
+        if "max_tokens" not in samp_param:
+            samp_param["max_tokens"] = 100
+        self.sampling_params = SamplingParams(**samp_param)
+        # self.sampling_params = SamplingParams(max_tokens=100, **job.get("sampling_params", {}))
         self.request_id = random_uuid()
         batch_size_growth_factor = job.get("batch_size_growth_factor")
         self.batch_size_growth_factor = float(batch_size_growth_factor) if batch_size_growth_factor else None 
@@ -47,11 +56,17 @@ class JobInput:
         self.min_batch_size = int(min_batch_size) if min_batch_size else None 
         self.openai_route = job.get("openai_route")
         self.openai_input = job.get("openai_input")
-
+class DummyState:
+    def __init__(self):
+        self.request_metadata = None
+        
 class DummyRequest:
+    def __init__(self):
+        self.headers = {}
+        self.state = DummyState()
     async def is_disconnected(self):
         return False
-    
+
 class BatchSize:
     def __init__(self, max_batch_size, min_batch_size, batch_size_growth_factor):
         self.max_batch_size = max_batch_size