Merge pull request #185 from runpod-workers/up-0.9.0

Version upgrade

.github/workflows/CI-runpod_dep.yml

@@ -19,32 +19,49 @@ jobs:
 
       - name: Check for new package version and update
         run: |
-          # Get current version
-          current_version=$(grep -oP 'runpod==\K[^"]+' ./builder/requirements.txt)
+          echo "Fetching the current runpod version from requirements.txt..."
+          
+          # Get current version, allowing both == and ~= in the search pattern
+          current_version=$(grep -oP 'runpod[~=]{1,2}\K[^"]+' ./builder/requirements.txt)
+          echo "Current version: $current_version"
 
-          # Get new version
+          # Extract major and minor from current version
+          current_major_minor=$(echo $current_version | cut -d. -f1,2)
+          echo "Current major.minor: $current_major_minor"
+
+          echo "Fetching the latest runpod version from PyPI..."
+          
+          # Get new version from PyPI
           new_version=$(curl -s https://pypi.org/pypi/runpod/json | jq -r .info.version)
           echo "NEW_VERSION_ENV=$new_version" >> $GITHUB_ENV
+          echo "New version: $new_version"
+
+          # Extract major and minor from new version
+          new_major_minor=$(echo $new_version | cut -d. -f1,2)
+          echo "New major.minor: $new_major_minor"
 
           if [ -z "$new_version" ]; then
-              echo "Failed to fetch the new version."
+              echo "ERROR: Failed to fetch the new version from PyPI."
               exit 1
           fi
 
-          # Check if the version is already up-to-date
-          if [ "$current_version" = "$new_version" ]; then
-              echo "The package version is already up-to-date."
+          # Check if the major or minor version is different
+          if [ "$current_major_minor" = "$new_major_minor" ]; then
+              echo "No update needed. The new version ($new_major_minor) is within the allowed range (~= $current_major_minor)."
               exit 0
           fi
 
-          # Update requirements.txt
-          sed -i "s/runpod==.*/runpod==$new_version/" ./builder/requirements.txt
+          echo "New major/minor detected ($new_major_minor). Updating requirements.txt..."
+
+          # Update requirements.txt, preserving the existing constraint type (~= or ==)
+          sed -i "s/runpod[~=][^ ]*/runpod~=$new_version/" ./builder/requirements.txt
+          echo "requirements.txt has been updated."
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v3
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: Update package version
+          commit-message: Update runpod package version
           title: Update runpod package version
           body: The package version has been updated to ${{ env.NEW_VERSION_ENV }}
           branch: runpod-package-update

.runpod/tests.json

@@ -0,0 +1,32 @@
+{
+    "tests": [
+        {
+            "name": "basic_inference_test",
+            "input": {
+                "prompt": "Write a short poem about artificial intelligence."
+            },
+            "timeout": 30000
+        }
+    ],
+    "config": {
+        "gpuTypeId": "NVIDIA GeForce RTX 4090",
+        "gpuCount": 1,
+        "env": [
+            {
+                "key": "MODEL_NAME",
+                "value": "facebook/opt-350m"
+            }
+        ],
+        "allowedCudaVersions": [
+            "12.7",
+            "12.6",
+            "12.5",
+            "12.4",
+            "12.3",
+            "12.2",
+            "12.1",
+            "12.0",
+            "11.7"
+        ]
+    }
+}

Dockerfile

@@ -12,7 +12,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     python3 -m pip install --upgrade -r /requirements.txt
 
 # Install vLLM (switching back to pip installs since issues that required building fork are fixed and space optimization is not as important since caching) and FlashInfer 
-RUN python3 -m pip install vllm==0.5.4 && \
+RUN python3 -m pip install vllm==0.9.0.1 && \
     python3 -m pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3
 
 # Setup for Option 2: Building the Image with the Model included
@@ -32,7 +32,7 @@ ENV MODEL_NAME=$MODEL_NAME \
     HF_DATASETS_CACHE="${BASE_PATH}/huggingface-cache/datasets" \
     HUGGINGFACE_HUB_CACHE="${BASE_PATH}/huggingface-cache/hub" \
     HF_HOME="${BASE_PATH}/huggingface-cache/hub" \
-    HF_HUB_ENABLE_HF_TRANSFER=1 
+    HF_HUB_ENABLE_HF_TRANSFER=0 
 
 ENV PYTHONPATH="/:/vllm-workspace"
 
@@ -40,11 +40,11 @@ ENV PYTHONPATH="/:/vllm-workspace"
 COPY src /src
 RUN --mount=type=secret,id=HF_TOKEN,required=false \
     if [ -f /run/secrets/HF_TOKEN ]; then \
-        export HF_TOKEN=$(cat /run/secrets/HF_TOKEN); \
+    export HF_TOKEN=$(cat /run/secrets/HF_TOKEN); \
     fi && \
     if [ -n "$MODEL_NAME" ]; then \
-        python3 /src/download_model.py; \
+    python3 /src/download_model.py; \
     fi
 
 # Start the handler
-CMD ["python3", "/src/handler.py"]
+CMD ["python3", "/src/handler.py"]

README.md

@@ -18,9 +18,9 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https:
 ### 1. UI for Deploying vLLM Worker on RunPod console:
 ![Demo of Deploying vLLM Worker on RunPod console with new UI](media/ui_demo.gif)
 
-### 2. Worker vLLM `v1.2.0` with vLLM `0.5.4` now available under `stable` tags 
+### 2. Worker vLLM `v2.6.0` with vLLM `0.9.0` now available under `stable` tags 
 
-Update v1.2.0 is now available, use the image tag `runpod/worker-v1-vllm:v1.2.0stable-cuda12.1.0`.
+Update v2.6.0 is now available, use the image tag `runpod/worker-v1-vllm:v2.6.0stable-cuda12.1.0`.
 
 ### 3. OpenAI-Compatible [Embedding Worker](https://github.com/runpod-workers/worker-infinity-embedding) Released
 Deploy your own OpenAI-compatible Serverless Endpoint on RunPod with multiple embedding models and fast inference for RAG and more! 
@@ -38,9 +38,8 @@ Worker vLLM is now cached on all RunPod machines, resulting in near-instant depl
     - [Environment Variables](#environment-variables)
       - [LLM Settings](#llm-settings)
       - [Tokenizer Settings](#tokenizer-settings)
-      - [Tensor Parallelism (Multi-GPU) Settings](#tensor-parallelism-multi-gpu-settings)
-      - [System Settings](#system-settings)
-      - [Streaming Batch Size](#streaming-batch-size)
+      - [System and Parallelism Settings](#system-and-parallelism-settings)
+      - [Streaming Batch Size Settings](#streaming-batch-size-settings)
       - [OpenAI Settings](#openai-settings)
       - [Serverless Settings](#serverless-settings)
   - [Option 2: Build Docker Image with Model Inside](#option-2-build-docker-image-with-model-inside)
@@ -82,7 +81,7 @@ Below is a summary of the available RunPod Worker images, categorized by image s
 
 | CUDA Version | Stable Image Tag                  | Development Image Tag             | Note                                                        |
 |--------------|-----------------------------------|-----------------------------------|----------------------------------------------------------------------|
-| 12.1.0       | `runpod/worker-v1-vllm:stable-cuda12.1.0` | `runpod/worker-v1-vllm:dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. |
+| 12.1.0       | `runpod/worker-v1-vllm:v2.6.0stable-cuda12.1.0` | `runpod/worker-v1-vllm:v2.6.0dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. |
 
 
 
@@ -91,9 +90,10 @@ Below is a summary of the available RunPod Worker images, categorized by image s
 #### Prerequisites
 - RunPod Account
 
-#### Environment Variables/Settings
+#### Environment Variables
 > Note:  `0` is equivalent to `False` and `1` is equivalent to `True` for boolean as int values.
 
+#### LLM Settings
 | `Name`                                    | `Default`             | `Type/Choices`                             | `Description` |
 |-------------------------------------------|-----------------------|--------------------------------------------|---------------|
 | `MODEL_NAME`                                   | 'facebook/opt-125m'   | `str`                                      | Name or path of the Hugging Face model to use. |
@@ -125,7 +125,7 @@ Below is a summary of the available RunPod Worker images, categorized by image s
 | `MAX_NUM_SEQS`                            | 256                   | `int`                                      | Maximum number of sequences per iteration. |
 | `MAX_LOGPROBS`                            | 20                    | `int`                                      | Max number of log probs to return when logprobs is specified in SamplingParams. |
 | `DISABLE_LOG_STATS`                       | False                 | `bool`                                     | Disable logging statistics. |
-| `QUANTIZATION`                            | None                  | ['awq', 'squeezellm', 'gptq']              | Method used to quantize the weights. |
+| `QUANTIZATION`                            | None                  | ['awq', 'squeezellm', 'gptq', 'bitsandbytes']              | Method used to quantize the weights. |
 | `ROPE_SCALING`                            | None                  | `dict`                                     | RoPE scaling configuration in JSON format. |
 | `ROPE_THETA`                              | None                  | `float`                                    | RoPE theta. Use with rope_scaling. |
 | `TOKENIZER_POOL_SIZE`                     | 0                     | `int`                                      | Size of tokenizer pool to use for asynchronous tokenization. |
@@ -139,6 +139,7 @@ Below is a summary of the available RunPod Worker images, categorized by image s
 | `LONG_LORA_SCALING_FACTORS`               | None                  | `tuple`                                    | Specify multiple scaling factors for LoRA adapters. |
 | `MAX_CPU_LORAS`                           | None                  | `int`                                      | Maximum number of LoRAs to store in CPU memory. |
 | `FULLY_SHARDED_LORAS`                     | False                 | `bool`                                     | Enable fully sharded LoRA layers. |
+| `LORA_MODULES`| `[]`| `list[dict]`| Add lora adapters from Hugging Face `[{"name": "xx", "path": "xxx/xxxx", "base_model_name": "xxx/xxxx"}`|
 | `SCHEDULER_DELAY_FACTOR`                  | 0.0                   | `float`                                    | Apply a delay before scheduling next prompt. |
 | `ENABLE_CHUNKED_PREFILL`                  | False                 | `bool`                                     | Enable chunked prefill requests. |
 | `SPECULATIVE_MODEL`                       | None                  | `str`                                      | The name of the draft model to be used in speculative decoding. |
@@ -157,11 +158,20 @@ Below is a summary of the available RunPod Worker images, categorized by image s
 | `PREEMPTION_CPU_CAPACITY`                 | 2                     | `float`                                    | The percentage of CPU memory used for the saved activations. |
 | `DISABLE_LOGGING_REQUEST`                 | False                 | `bool`                                     | Disable logging requests. |
 | `MAX_LOG_LEN`                             | None                  | `int`                                      | Max number of prompt characters or prompt ID numbers being printed in log. |
-**Tokenizer Settings**
+
+
+#### Tokenizer Settings
+
+| `Name`                                    | `Default`             | `Type/Choices`                             | `Description` |
+|-------------------------------------------|-----------------------|--------------------------------------------|---------------|
 | `TOKENIZER_NAME`                    | `None`               | `str`                                         |Tokenizer repository to use a different tokenizer than the model's default. |
 | `TOKENIZER_REVISION`                | `None`               | `str`                                         |Tokenizer revision to load. |
 | `CUSTOM_CHAT_TEMPLATE`              | `None`               | `str` of single-line jinja template                                         |Custom chat jinja template. [More Info](https://huggingface.co/docs/transformers/chat_templating) |
-**System, GPU, and Tensor Parallelism(Multi-GPU) Settings**
+
+#### System and Parallelism Settings
+
+| `Name`                                    | `Default`             | `Type/Choices`                             | `Description` |
+|-------------------------------------------|-----------------------|--------------------------------------------|---------------|
 | `GPU_MEMORY_UTILIZATION`            | `0.95`               | `float`                                         |Sets GPU VRAM utilization. |
 | `MAX_PARALLEL_LOADING_WORKERS`      | `None`               | `int`                                         |Load model sequentially in multiple batches, to avoid RAM OOM when using tensor parallel and large models. |
 | `BLOCK_SIZE`                        | `16`                 | `8`, `16`, `32`                           |Token block size for contiguous chunks of tokens. |
@@ -169,16 +179,31 @@ Below is a summary of the available RunPod Worker images, categorized by image s
 | `ENFORCE_EAGER`                     | False                  | `bool`                                         |Always use eager-mode PyTorch. If False(`0`), will use eager mode and CUDA graph in hybrid for maximal performance and flexibility. |
 | `MAX_SEQ_LEN_TO_CAPTURE`        | `8192`               | `int`                                     |Maximum context length covered by CUDA graphs. When a sequence has context length larger than this, we fall back to eager mode.|
 | `DISABLE_CUSTOM_ALL_REDUCE`         | `0`                  | `int`                                         |Enables or disables custom all reduce. |
-**Streaming Batch Size Settings**:  
+
+
+#### Streaming Batch Size Settings
+
+The way this works is that the first request will have a batch size of `DEFAULT_MIN_BATCH_SIZE`, and each subsequent request will have a batch size of `previous_batch_size * DEFAULT_BATCH_SIZE_GROWTH_FACTOR`. This will continue until the batch size reaches `DEFAULT_BATCH_SIZE`. E.g. for the default values, the batch sizes will be `1, 3, 9, 27, 50, 50, 50, ...`. You can also specify this per request, with inputs `max_batch_size`, `min_batch_size`, and `batch_size_growth_factor`. This has nothing to do with vLLM's internal batching, but rather the number of tokens sent in each HTTP request from the worker
+
+
+| `Name`                                    | `Default`             | `Type/Choices`                             | `Description` |
+|-------------------------------------------|-----------------------|--------------------------------------------|---------------|
 | `DEFAULT_BATCH_SIZE`                | `50`                 | `int`                                         |Default and Maximum batch size for token streaming to reduce HTTP calls. |
 | `DEFAULT_MIN_BATCH_SIZE`            | `1`                  | `int`                                         |Batch size for the first request, which will be multiplied by the growth factor every subsequent request. |
 | `DEFAULT_BATCH_SIZE_GROWTH_FACTOR`  | `3`                  | `float`                                         |Growth factor for dynamic batch size. |
-The way this works is that the first request will have a batch size of `DEFAULT_MIN_BATCH_SIZE`, and each subsequent request will have a batch size of `previous_batch_size * DEFAULT_BATCH_SIZE_GROWTH_FACTOR`. This will continue until the batch size reaches `DEFAULT_BATCH_SIZE`. E.g. for the default values, the batch sizes will be `1, 3, 9, 27, 50, 50, 50, ...`. You can also specify this per request, with inputs `max_batch_size`, `min_batch_size`, and `batch_size_growth_factor`. This has nothing to do with vLLM's internal batching, but rather the number of tokens sent in each HTTP request from the worker |
-**OpenAI Settings**
+
+#### OpenAI Settings
+
+| `Name`                                    | `Default`             | `Type/Choices`                             | `Description` |
+|-------------------------------------------|-----------------------|--------------------------------------------|---------------|
 | `RAW_OPENAI_OUTPUT`                 | `1`                  | boolean as `int`                                         |Enables raw OpenAI SSE format string output when streaming.  **Required** to be enabled (which it is by default) for OpenAI compatibility. |
 | `OPENAI_SERVED_MODEL_NAME_OVERRIDE` | `None`               | `str`                                         |Overrides the name of the served model from model repo/path to specified name, which you will then be able to use the value for the `model` parameter when making OpenAI requests |
 | `OPENAI_RESPONSE_ROLE`              | `assistant`          | `str`                       |Role of the LLM's Response in OpenAI Chat Completions. |
-**Serverless Settings**
+
+#### Serverless Settings
+
+| `Name`                                    | `Default`             | `Type/Choices`                             | `Description` |
+|-------------------------------------------|-----------------------|--------------------------------------------|---------------|
 | `MAX_CONCURRENCY`                   | `300`                | `int`                                         |Max concurrent requests per worker. vLLM has an internal queue, so you don't have to worry about limiting by VRAM, this is for improving scaling/load balancing efficiency |
 | `DISABLE_LOG_STATS`                 | False                  | `bool`                                         |Enables or disables vLLM stats logging. |
 | `DISABLE_LOG_REQUESTS`              | False                  | `bool`                                         |Enables or disables vLLM request logging. |
@@ -488,7 +513,15 @@ The prompt string can be any string, and the model's chat template will not be a
 
     Example:
     ```json
-    "prompt": "..."
+    {
+      "input": {
+        "prompt": "why sky is blue?",
+        "sampling_params": {
+          "temperature": 0.7,
+          "max_tokens": 100
+        }
+      }
+    }
     ```
 2. `messages`
 Your list can contain any number of messages, and each message usually can have any role from the following list:
@@ -502,20 +535,28 @@ Your list can contain any number of messages, and each message usually can have
 
     Example:
     ```json
-    "messages": [
-        {
-          "role": "system",
-          "content": "..."
-        },
-        {
-          "role": "user",
-          "content": "..."
-        },
-        {
-          "role": "assistant",
-          "content": "..."
+    {
+      "input": {
+        "messages": [
+          {
+            "role": "system",
+            "content": "You are a helpful AI assistant that provides clear and concise responses."
+          },
+          {
+            "role": "user",
+            "content": "Can you explain the difference between supervised and unsupervised learning?"
+          },
+          {
+            "role": "assistant",
+            "content": "Sure! Supervised learning uses labeled data, meaning each input has a corresponding correct output. The model learns by mapping inputs to known outputs. In contrast, unsupervised learning works with unlabeled data, where the model identifies patterns, structures, or clusters without predefined answers."
+          }
+        ],
+        "sampling_params": {
+          "temperature": 0.7,
+          "max_tokens": 100
         }
-      ]
+      }
+    }
     ```
 
 </details>
@@ -600,4 +641,4 @@ The JSON consists of two main parts, schema and versions.
     }
   }
 }
-```
+```

builder/requirements.txt

@@ -1,10 +1,12 @@
 ray
 pandas
 pyarrow
-runpod==1.7.0
+runpod~=1.7.7
 huggingface-hub
 packaging
-typing-extensions==4.7.1
+typing-extensions>=4.8.0
 pydantic
 pydantic-settings
-hf-transfer
+hf-transfer
+transformers
+bitsandbytes>=0.45.0

docker-bake.hcl

@@ -7,7 +7,7 @@ variable "REPOSITORY" {
 }
 
 variable "BASE_IMAGE_VERSION" {
-  default = "stable"
+  default = "v2.6.0stable"
 }
 
 group "all" {

src/engine.py

@@ -4,13 +4,16 @@ import json
 import asyncio
 
 from dotenv import load_dotenv
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Optional
 import time
 
 from vllm import AsyncLLMEngine
+from vllm.entrypoints.logger import RequestLogger
 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
 from vllm.entrypoints.openai.protocol import ChatCompletionRequest, CompletionRequest, ErrorResponse
+from vllm.entrypoints.openai.serving_models import BaseModelPath, LoRAModulePath, OpenAIServingModels
+
 
 from utils import DummyRequest, JobInput, BatchSize, create_error_response
 from constants import DEFAULT_MAX_CONCURRENCY, DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE_GROWTH_FACTOR, DEFAULT_MIN_BATCH_SIZE
@@ -119,29 +122,61 @@ class OpenAIvLLMEngine(vLLMEngine):
         super().__init__(vllm_engine)
         self.served_model_name = os.getenv("OPENAI_SERVED_MODEL_NAME_OVERRIDE") or self.engine_args.model
         self.response_role = os.getenv("OPENAI_RESPONSE_ROLE") or "assistant"
+        self.lora_adapters = self._load_lora_adapters()
         asyncio.run(self._initialize_engines())
         self.raw_openai_output = bool(int(os.getenv("RAW_OPENAI_OUTPUT", 1)))
 
+    def _load_lora_adapters(self):
+        adapters = []
+        try:
+            adapters = json.loads(os.getenv("LORA_MODULES", '[]'))
+        except Exception as e:
+            logging.info(f"---Initialized adapter json load error: {e}")
+
+        for i, adapter in enumerate(adapters):
+            try:
+                adapters[i] = LoRAModulePath(**adapter)
+                logging.info(f"---Initialized adapter: {adapter}")
+            except Exception as e:
+                logging.info(f"---Initialized adapter not worked: {e}")
+                continue
+        return adapters
+
     async def _initialize_engines(self):
         self.model_config = await self.llm.get_model_config()
-        
-        self.chat_engine = OpenAIServingChat(
-            async_engine_client=self.llm, 
+        self.base_model_paths = [
+            BaseModelPath(name=self.engine_args.model, model_path=self.engine_args.model)
+        ]
+
+        self.serving_models = OpenAIServingModels(
+            engine_client=self.llm,
             model_config=self.model_config,
-            served_model_names=[self.served_model_name], 
-            response_role=self.response_role,
-            chat_template=self.tokenizer.tokenizer.chat_template,
-            lora_modules=None,
+            base_model_paths=self.base_model_paths,
+            lora_modules=self.lora_adapters,
             prompt_adapters=None,
-            request_logger=None
+        )
+        await self.serving_models.init_static_loras()
+        self.chat_engine = OpenAIServingChat(
+            engine_client=self.llm, 
+            model_config=self.model_config,
+            models=self.serving_models,
+            response_role=self.response_role,
+            request_logger=None,
+            chat_template=self.tokenizer.tokenizer.chat_template,
+            chat_template_content_format="auto",
+            # enable_reasoning=os.getenv('ENABLE_REASONING', 'false').lower() == 'true',
+            reasoning_parser= os.getenv('REASONING_PARSER', "") or None,
+            # return_token_as_token_ids=False,
+            enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
+            tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None,
+            enable_prompt_tokens_details=False
         )
         self.completion_engine = OpenAIServingCompletion(
-            async_engine_client=self.llm, 
+            engine_client=self.llm, 
             model_config=self.model_config,
-            served_model_names=[self.served_model_name],
-            lora_modules=[],
-            prompt_adapters=None,
-            request_logger=None
+            models=self.serving_models,
+            request_logger=None,
+            # return_token_as_token_ids=False,
         )
     
     async def generate(self, openai_request: JobInput):
@@ -154,10 +189,7 @@ class OpenAIvLLMEngine(vLLMEngine):
             yield create_error_response("Invalid route").model_dump()
     
     async def _handle_model_request(self):
-        models = await self.chat_engine.show_available_models()
-        fixed_model = models.data[0]
-        fixed_model.id = self.served_model_name
-        models.data = [fixed_model]
+        models = await self.serving_models.show_available_models()
         return models.model_dump()
     
     async def _handle_chat_or_completion_request(self, openai_request: JobInput):
@@ -176,7 +208,8 @@ class OpenAIvLLMEngine(vLLMEngine):
             yield create_error_response(str(e)).model_dump()
             return
         
-        response_generator = await generator_function(request, raw_request=None)
+        dummy_request = DummyRequest()
+        response_generator = await generator_function(request, raw_request=dummy_request)
 
         if not openai_request.openai_input.get("stream") or isinstance(response_generator, ErrorResponse):
             yield response_generator.model_dump()

src/engine_args.py

@@ -4,6 +4,7 @@ import logging
 from torch.cuda import device_count
 from vllm import AsyncEngineArgs
 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
+from src.utils import convert_limit_mm_per_prompt
 
 RENAME_ARGS_MAP = {
     "MODEL_NAME": "model",
@@ -88,7 +89,8 @@ DEFAULT_ARGS = {
     "typical_acceptance_sampler_posterior_alpha": float(os.getenv('TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA', 0)) or None,
     "qlora_adapter_name_or_path": os.getenv('QLORA_ADAPTER_NAME_OR_PATH', None),
     "disable_logprobs_during_spec_decoding": os.getenv('DISABLE_LOGPROBS_DURING_SPEC_DECODING', None),
-    "otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None)
+    "otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None),
+    "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true'),
 }
 
 def match_vllm_args(args):
@@ -120,7 +122,7 @@ def get_local_args():
         local_args = json.load(f)
 
     if local_args.get("MODEL_NAME") is None:
-        raise ValueError("Model name not found in /local_model_args.json. There was a problem when baking the model in.")
+        logging.warning("Model name not found in /local_model_args.json. There maybe was a problem when baking the model in.")
 
     logging.info(f"Using baked in model with args: {local_args}")
     os.environ["TRANSFORMERS_OFFLINE"] = "1"
@@ -145,6 +147,9 @@ def get_engine_args():
     
     # Rename and match to vllm args
     args = match_vllm_args(args)
+
+    if args.get("load_format") == "bitsandbytes":
+        args["quantization"] = args["load_format"]
     
     # Set tensor parallel size and max parallel loading workers if more than 1 GPU is available
     num_gpus = device_count()

src/utils.py

@@ -3,6 +3,7 @@ import logging
 from http import HTTPStatus
 from functools import wraps
 from time import time
+from vllm.entrypoints.openai.protocol import RequestResponseMetadata
 
 try:
     from vllm.utils import random_uuid
@@ -14,6 +15,10 @@ except ImportError:
 
 logging.basicConfig(level=logging.INFO)
 
+def convert_limit_mm_per_prompt(input_string: str):
+    key, value = input_string.split('=')
+    return {key: int(value)}
+
 def count_physical_cores():
     with open('/proc/cpuinfo') as f:
         content = f.readlines()
@@ -39,7 +44,11 @@ class JobInput:
         self.max_batch_size = job.get("max_batch_size")
         self.apply_chat_template = job.get("apply_chat_template", False)
         self.use_openai_format = job.get("use_openai_format", False)
-        self.sampling_params = SamplingParams(**job.get("sampling_params", {}))
+        samp_param = job.get("sampling_params", {})
+        if "max_tokens" not in samp_param:
+            samp_param["max_tokens"] = 100
+        self.sampling_params = SamplingParams(**samp_param)
+        # self.sampling_params = SamplingParams(max_tokens=100, **job.get("sampling_params", {}))
         self.request_id = random_uuid()
         batch_size_growth_factor = job.get("batch_size_growth_factor")
         self.batch_size_growth_factor = float(batch_size_growth_factor) if batch_size_growth_factor else None 
@@ -47,11 +56,17 @@ class JobInput:
         self.min_batch_size = int(min_batch_size) if min_batch_size else None 
         self.openai_route = job.get("openai_route")
         self.openai_input = job.get("openai_input")
-
+class DummyState:
+    def __init__(self):
+        self.request_metadata = None
+        
 class DummyRequest:
+    def __init__(self):
+        self.headers = {}
+        self.state = DummyState()
     async def is_disconnected(self):
         return False
-    
+
 class BatchSize:
     def __init__(self, max_batch_size, min_batch_size, batch_size_growth_factor):
         self.max_batch_size = max_batch_size