Merge pull request #185 from runpod-workers/up-0.9.0

Version upgrade

.runpod/tests.json

@@ -1,44 +1,32 @@
 {
-  "tests": [
+    "tests": [
-    {
+        {
-      "name": "basic_inference_test",
+            "name": "basic_inference_test",
-      "input": {
+            "input": {
-        "prompt": "Write a short poem about artificial intelligence.",
+                "prompt": "Write a short poem about artificial intelligence."
-      },
+            },
-      "timeout": 30000
+            "timeout": 30000
-    }
+        }
-  ],
-  "config": {
-    "gpuTypeId": "NVIDIA GeForce RTX 4090",
-    "gpuCount": 1,
-    "env": [
-      {
-        "key": "MODEL_NAME",
-        "value": "facebook/opt-350m"
-      },
-      {
-        "key": "HF_TOKEN",
-        "value": "hf_dummy_token_for_testing_purposes_only"
-      },
-      {
-        "key": "MAX_MODEL_LEN",
-        "value": "8192"
-      },
-      {
-        "key": "GPU_MEMORY_UTILIZATION",
-        "value": "0.95"
-      }
     ],
-    "allowedCudaVersions": [
+    "config": {
-      "12.7", 
+        "gpuTypeId": "NVIDIA GeForce RTX 4090",
-      "12.6", 
+        "gpuCount": 1,
-      "12.5", 
+        "env": [
-      "12.4", 
+            {
-      "12.3", 
+                "key": "MODEL_NAME",
-      "12.2", 
+                "value": "facebook/opt-350m"
-      "12.1", 
+            }
-      "12.0", 
+        ],
-      "11.7"
+        "allowedCudaVersions": [
-    ]
+            "12.7",
-  }
+            "12.6",
+            "12.5",
+            "12.4",
+            "12.3",
+            "12.2",
+            "12.1",
+            "12.0",
+            "11.7"
+        ]
+    }
 }

Dockerfile

@@ -12,7 +12,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     python3 -m pip install --upgrade -r /requirements.txt
 
 # Install vLLM (switching back to pip installs since issues that required building fork are fixed and space optimization is not as important since caching) and FlashInfer 
-RUN python3 -m pip install vllm==0.8.2 && \
+RUN python3 -m pip install vllm==0.9.0.1 && \
     python3 -m pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3
 
 # Setup for Option 2: Building the Image with the Model included
@@ -47,4 +47,4 @@ RUN --mount=type=secret,id=HF_TOKEN,required=false \
     fi
 
 # Start the handler
-CMD ["python3", "/src/handler.py"]
+CMD ["python3", "/src/handler.py"]

README.md

@@ -18,9 +18,9 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https:
 ### 1. UI for Deploying vLLM Worker on RunPod console:
 ![Demo of Deploying vLLM Worker on RunPod console with new UI](media/ui_demo.gif)
 
-### 2. Worker vLLM `v2.2.0` with vLLM `0.8.2` now available under `stable` tags 
+### 2. Worker vLLM `v2.6.0` with vLLM `0.9.0` now available under `stable` tags 
 
-Update v2.2.0 is now available, use the image tag `runpod/worker-v1-vllm:v2.2.0stable-cuda12.1.0`.
+Update v2.6.0 is now available, use the image tag `runpod/worker-v1-vllm:v2.6.0stable-cuda12.1.0`.
 
 ### 3. OpenAI-Compatible [Embedding Worker](https://github.com/runpod-workers/worker-infinity-embedding) Released
 Deploy your own OpenAI-compatible Serverless Endpoint on RunPod with multiple embedding models and fast inference for RAG and more! 
@@ -38,9 +38,8 @@ Worker vLLM is now cached on all RunPod machines, resulting in near-instant depl
     - [Environment Variables](#environment-variables)
       - [LLM Settings](#llm-settings)
       - [Tokenizer Settings](#tokenizer-settings)
-      - [Tensor Parallelism (Multi-GPU) Settings](#tensor-parallelism-multi-gpu-settings)
+      - [System and Parallelism Settings](#system-and-parallelism-settings)
-      - [System Settings](#system-settings)
+      - [Streaming Batch Size Settings](#streaming-batch-size-settings)
-      - [Streaming Batch Size](#streaming-batch-size)
       - [OpenAI Settings](#openai-settings)
       - [Serverless Settings](#serverless-settings)
   - [Option 2: Build Docker Image with Model Inside](#option-2-build-docker-image-with-model-inside)
@@ -82,7 +81,7 @@ Below is a summary of the available RunPod Worker images, categorized by image s
 
 | CUDA Version | Stable Image Tag                  | Development Image Tag             | Note                                                        |
 |--------------|-----------------------------------|-----------------------------------|----------------------------------------------------------------------|
-| 12.1.0       | `runpod/worker-v1-vllm:v2.2.0stable-cuda12.1.0` | `runpod/worker-v1-vllm:v2.2.0dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. |
+| 12.1.0       | `runpod/worker-v1-vllm:v2.6.0stable-cuda12.1.0` | `runpod/worker-v1-vllm:v2.6.0dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. |
 
 
 
@@ -91,9 +90,10 @@ Below is a summary of the available RunPod Worker images, categorized by image s
 #### Prerequisites
 - RunPod Account
 
-#### Environment Variables/Settings
+#### Environment Variables
 > Note:  `0` is equivalent to `False` and `1` is equivalent to `True` for boolean as int values.
 
+#### LLM Settings
 | `Name`                                    | `Default`             | `Type/Choices`                             | `Description` |
 |-------------------------------------------|-----------------------|--------------------------------------------|---------------|
 | `MODEL_NAME`                                   | 'facebook/opt-125m'   | `str`                                      | Name or path of the Hugging Face model to use. |
@@ -125,7 +125,7 @@ Below is a summary of the available RunPod Worker images, categorized by image s
 | `MAX_NUM_SEQS`                            | 256                   | `int`                                      | Maximum number of sequences per iteration. |
 | `MAX_LOGPROBS`                            | 20                    | `int`                                      | Max number of log probs to return when logprobs is specified in SamplingParams. |
 | `DISABLE_LOG_STATS`                       | False                 | `bool`                                     | Disable logging statistics. |
-| `QUANTIZATION`                            | None                  | ['awq', 'squeezellm', 'gptq']              | Method used to quantize the weights. |
+| `QUANTIZATION`                            | None                  | ['awq', 'squeezellm', 'gptq', 'bitsandbytes']              | Method used to quantize the weights. |
 | `ROPE_SCALING`                            | None                  | `dict`                                     | RoPE scaling configuration in JSON format. |
 | `ROPE_THETA`                              | None                  | `float`                                    | RoPE theta. Use with rope_scaling. |
 | `TOKENIZER_POOL_SIZE`                     | 0                     | `int`                                      | Size of tokenizer pool to use for asynchronous tokenization. |
@@ -139,6 +139,7 @@ Below is a summary of the available RunPod Worker images, categorized by image s
 | `LONG_LORA_SCALING_FACTORS`               | None                  | `tuple`                                    | Specify multiple scaling factors for LoRA adapters. |
 | `MAX_CPU_LORAS`                           | None                  | `int`                                      | Maximum number of LoRAs to store in CPU memory. |
 | `FULLY_SHARDED_LORAS`                     | False                 | `bool`                                     | Enable fully sharded LoRA layers. |
+| `LORA_MODULES`| `[]`| `list[dict]`| Add lora adapters from Hugging Face `[{"name": "xx", "path": "xxx/xxxx", "base_model_name": "xxx/xxxx"}`|
 | `SCHEDULER_DELAY_FACTOR`                  | 0.0                   | `float`                                    | Apply a delay before scheduling next prompt. |
 | `ENABLE_CHUNKED_PREFILL`                  | False                 | `bool`                                     | Enable chunked prefill requests. |
 | `SPECULATIVE_MODEL`                       | None                  | `str`                                      | The name of the draft model to be used in speculative decoding. |
@@ -157,11 +158,20 @@ Below is a summary of the available RunPod Worker images, categorized by image s
 | `PREEMPTION_CPU_CAPACITY`                 | 2                     | `float`                                    | The percentage of CPU memory used for the saved activations. |
 | `DISABLE_LOGGING_REQUEST`                 | False                 | `bool`                                     | Disable logging requests. |
 | `MAX_LOG_LEN`                             | None                  | `int`                                      | Max number of prompt characters or prompt ID numbers being printed in log. |
-**Tokenizer Settings**
+
+
+#### Tokenizer Settings
+
+| `Name`                                    | `Default`             | `Type/Choices`                             | `Description` |
+|-------------------------------------------|-----------------------|--------------------------------------------|---------------|
 | `TOKENIZER_NAME`                    | `None`               | `str`                                         |Tokenizer repository to use a different tokenizer than the model's default. |
 | `TOKENIZER_REVISION`                | `None`               | `str`                                         |Tokenizer revision to load. |
 | `CUSTOM_CHAT_TEMPLATE`              | `None`               | `str` of single-line jinja template                                         |Custom chat jinja template. [More Info](https://huggingface.co/docs/transformers/chat_templating) |
-**System, GPU, and Tensor Parallelism(Multi-GPU) Settings**
+
+#### System and Parallelism Settings
+
+| `Name`                                    | `Default`             | `Type/Choices`                             | `Description` |
+|-------------------------------------------|-----------------------|--------------------------------------------|---------------|
 | `GPU_MEMORY_UTILIZATION`            | `0.95`               | `float`                                         |Sets GPU VRAM utilization. |
 | `MAX_PARALLEL_LOADING_WORKERS`      | `None`               | `int`                                         |Load model sequentially in multiple batches, to avoid RAM OOM when using tensor parallel and large models. |
 | `BLOCK_SIZE`                        | `16`                 | `8`, `16`, `32`                           |Token block size for contiguous chunks of tokens. |
@@ -169,16 +179,31 @@ Below is a summary of the available RunPod Worker images, categorized by image s
 | `ENFORCE_EAGER`                     | False                  | `bool`                                         |Always use eager-mode PyTorch. If False(`0`), will use eager mode and CUDA graph in hybrid for maximal performance and flexibility. |
 | `MAX_SEQ_LEN_TO_CAPTURE`        | `8192`               | `int`                                     |Maximum context length covered by CUDA graphs. When a sequence has context length larger than this, we fall back to eager mode.|
 | `DISABLE_CUSTOM_ALL_REDUCE`         | `0`                  | `int`                                         |Enables or disables custom all reduce. |
-**Streaming Batch Size Settings**:  
+
+
+#### Streaming Batch Size Settings
+
+The way this works is that the first request will have a batch size of `DEFAULT_MIN_BATCH_SIZE`, and each subsequent request will have a batch size of `previous_batch_size * DEFAULT_BATCH_SIZE_GROWTH_FACTOR`. This will continue until the batch size reaches `DEFAULT_BATCH_SIZE`. E.g. for the default values, the batch sizes will be `1, 3, 9, 27, 50, 50, 50, ...`. You can also specify this per request, with inputs `max_batch_size`, `min_batch_size`, and `batch_size_growth_factor`. This has nothing to do with vLLM's internal batching, but rather the number of tokens sent in each HTTP request from the worker
+
+
+| `Name`                                    | `Default`             | `Type/Choices`                             | `Description` |
+|-------------------------------------------|-----------------------|--------------------------------------------|---------------|
 | `DEFAULT_BATCH_SIZE`                | `50`                 | `int`                                         |Default and Maximum batch size for token streaming to reduce HTTP calls. |
 | `DEFAULT_MIN_BATCH_SIZE`            | `1`                  | `int`                                         |Batch size for the first request, which will be multiplied by the growth factor every subsequent request. |
 | `DEFAULT_BATCH_SIZE_GROWTH_FACTOR`  | `3`                  | `float`                                         |Growth factor for dynamic batch size. |
-The way this works is that the first request will have a batch size of `DEFAULT_MIN_BATCH_SIZE`, and each subsequent request will have a batch size of `previous_batch_size * DEFAULT_BATCH_SIZE_GROWTH_FACTOR`. This will continue until the batch size reaches `DEFAULT_BATCH_SIZE`. E.g. for the default values, the batch sizes will be `1, 3, 9, 27, 50, 50, 50, ...`. You can also specify this per request, with inputs `max_batch_size`, `min_batch_size`, and `batch_size_growth_factor`. This has nothing to do with vLLM's internal batching, but rather the number of tokens sent in each HTTP request from the worker |
+
-**OpenAI Settings**
+#### OpenAI Settings
+
+| `Name`                                    | `Default`             | `Type/Choices`                             | `Description` |
+|-------------------------------------------|-----------------------|--------------------------------------------|---------------|
 | `RAW_OPENAI_OUTPUT`                 | `1`                  | boolean as `int`                                         |Enables raw OpenAI SSE format string output when streaming.  **Required** to be enabled (which it is by default) for OpenAI compatibility. |
 | `OPENAI_SERVED_MODEL_NAME_OVERRIDE` | `None`               | `str`                                         |Overrides the name of the served model from model repo/path to specified name, which you will then be able to use the value for the `model` parameter when making OpenAI requests |
 | `OPENAI_RESPONSE_ROLE`              | `assistant`          | `str`                       |Role of the LLM's Response in OpenAI Chat Completions. |
-**Serverless Settings**
+
+#### Serverless Settings
+
+| `Name`                                    | `Default`             | `Type/Choices`                             | `Description` |
+|-------------------------------------------|-----------------------|--------------------------------------------|---------------|
 | `MAX_CONCURRENCY`                   | `300`                | `int`                                         |Max concurrent requests per worker. vLLM has an internal queue, so you don't have to worry about limiting by VRAM, this is for improving scaling/load balancing efficiency |
 | `DISABLE_LOG_STATS`                 | False                  | `bool`                                         |Enables or disables vLLM stats logging. |
 | `DISABLE_LOG_REQUESTS`              | False                  | `bool`                                         |Enables or disables vLLM request logging. |

builder/requirements.txt

@@ -4,8 +4,9 @@ pyarrow
 runpod~=1.7.7
 huggingface-hub
 packaging
-typing-extensions==4.7.1
+typing-extensions>=4.8.0
 pydantic
 pydantic-settings
 hf-transfer
 transformers
+bitsandbytes>=0.45.0

docker-bake.hcl

@@ -7,7 +7,7 @@ variable "REPOSITORY" {
 }
 
 variable "BASE_IMAGE_VERSION" {
-  default = "v2.0.0stable"
+  default = "v2.6.0stable"
 }
 
 group "all" {

src/engine.py

@@ -122,31 +122,40 @@ class OpenAIvLLMEngine(vLLMEngine):
         super().__init__(vllm_engine)
         self.served_model_name = os.getenv("OPENAI_SERVED_MODEL_NAME_OVERRIDE") or self.engine_args.model
         self.response_role = os.getenv("OPENAI_RESPONSE_ROLE") or "assistant"
+        self.lora_adapters = self._load_lora_adapters()
         asyncio.run(self._initialize_engines())
         self.raw_openai_output = bool(int(os.getenv("RAW_OPENAI_OUTPUT", 1)))
-        
+
+    def _load_lora_adapters(self):
+        adapters = []
+        try:
+            adapters = json.loads(os.getenv("LORA_MODULES", '[]'))
+        except Exception as e:
+            logging.info(f"---Initialized adapter json load error: {e}")
+
+        for i, adapter in enumerate(adapters):
+            try:
+                adapters[i] = LoRAModulePath(**adapter)
+                logging.info(f"---Initialized adapter: {adapter}")
+            except Exception as e:
+                logging.info(f"---Initialized adapter not worked: {e}")
+                continue
+        return adapters
+
     async def _initialize_engines(self):
         self.model_config = await self.llm.get_model_config()
         self.base_model_paths = [
             BaseModelPath(name=self.engine_args.model, model_path=self.engine_args.model)
         ]
 
-        lora_modules = os.getenv('LORA_MODULES', None)
-        if lora_modules is not None:
-            try:
-                lora_modules = json.loads(lora_modules)
-                lora_modules = [LoRAModulePath(**lora_modules)]
-            except:
-                lora_modules = None
-
         self.serving_models = OpenAIServingModels(
             engine_client=self.llm,
             model_config=self.model_config,
             base_model_paths=self.base_model_paths,
-            lora_modules=None,
+            lora_modules=self.lora_adapters,
             prompt_adapters=None,
         )
-
+        await self.serving_models.init_static_loras()
         self.chat_engine = OpenAIServingChat(
             engine_client=self.llm, 
             model_config=self.model_config,
@@ -156,7 +165,7 @@ class OpenAIvLLMEngine(vLLMEngine):
             chat_template=self.tokenizer.tokenizer.chat_template,
             chat_template_content_format="auto",
             # enable_reasoning=os.getenv('ENABLE_REASONING', 'false').lower() == 'true',
-            # reasoning_parser=None,
+            reasoning_parser= os.getenv('REASONING_PARSER', "") or None,
             # return_token_as_token_ids=False,
             enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
             tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None,

src/engine_args.py

@@ -122,7 +122,7 @@ def get_local_args():
         local_args = json.load(f)
 
     if local_args.get("MODEL_NAME") is None:
-        raise ValueError("Model name not found in /local_model_args.json. There was a problem when baking the model in.")
+        logging.warning("Model name not found in /local_model_args.json. There maybe was a problem when baking the model in.")
 
     logging.info(f"Using baked in model with args: {local_args}")
     os.environ["TRANSFORMERS_OFFLINE"] = "1"
@@ -147,6 +147,9 @@ def get_engine_args():
     
     # Rename and match to vllm args
     args = match_vllm_args(args)
+
+    if args.get("load_format") == "bitsandbytes":
+        args["quantization"] = args["load_format"]
     
     # Set tensor parallel size and max parallel loading workers if more than 1 GPU is available
     num_gpus = device_count()

worker-config.json

@@ -1,5 +1,249 @@
 {
   "versions": {
+    "0.9.0": {
+      "imageName": "runpod/worker-v1-vllm:v2.6.0stable-cuda12.1.0",
+      "minimumCudaVersion": "12.1",
+      "categories": [
+        {
+          "title": "LLM Settings",
+          "settings": [
+            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
+            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
+            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
+            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
+            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
+            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
+            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
+            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
+            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
+            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
+            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
+            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
+            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
+            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
+            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
+            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
+            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
+            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
+            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
+          ]
+        },
+        {
+          "title": "Tokenizer Settings",
+          "settings": [
+            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
+          ]
+        },
+        {
+          "title": "System Settings",
+          "settings": [
+            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
+            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
+          ]
+        },
+        {
+          "title": "Streaming Settings",
+          "settings": [
+            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
+          ]
+        },
+        {
+          "title": "OpenAI Settings",
+          "settings": [
+            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
+          ]
+        },
+        {
+          "title": "Serverless Settings",
+          "settings": [
+            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
+          ]
+        }
+      ]
+    },
+    "0.8.5": {
+      "imageName": "runpod/worker-v1-vllm:v2.5.0stable-cuda12.1.0",
+      "minimumCudaVersion": "12.1",
+      "categories": [
+        {
+          "title": "LLM Settings",
+          "settings": [
+            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
+            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
+            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
+            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
+            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
+            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
+            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
+            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
+            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
+            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
+            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
+            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
+            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
+            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
+            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
+            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
+            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
+            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
+            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
+          ]
+        },
+        {
+          "title": "Tokenizer Settings",
+          "settings": [
+            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
+          ]
+        },
+        {
+          "title": "System Settings",
+          "settings": [
+            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
+            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
+          ]
+        },
+        {
+          "title": "Streaming Settings",
+          "settings": [
+            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
+          ]
+        },
+        {
+          "title": "OpenAI Settings",
+          "settings": [
+            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
+          ]
+        },
+        {
+          "title": "Serverless Settings",
+          "settings": [
+            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
+          ]
+        }
+      ]
+    },
+    "0.8.4": {
+      "imageName": "runpod/worker-v1-vllm:v2.4.0stable-cuda12.1.0",
+      "minimumCudaVersion": "12.1",
+      "categories": [
+        {
+          "title": "LLM Settings",
+          "settings": [
+            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
+            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
+            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
+            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
+            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
+            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
+            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
+            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
+            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
+            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
+            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
+            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
+            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
+            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
+            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
+            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
+            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
+            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
+            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
+          ]
+        },
+        {
+          "title": "Tokenizer Settings",
+          "settings": [
+            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
+          ]
+        },
+        {
+          "title": "System Settings",
+          "settings": [
+            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
+            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
+          ]
+        },
+        {
+          "title": "Streaming Settings",
+          "settings": [
+            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
+          ]
+        },
+        {
+          "title": "OpenAI Settings",
+          "settings": [
+            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
+          ]
+        },
+        {
+          "title": "Serverless Settings",
+          "settings": [
+            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
+          ]
+        }
+      ]
+    },
+    "0.8.3": {
+      "imageName": "runpod/worker-v1-vllm:v2.3.0stable-cuda12.1.0",
+      "minimumCudaVersion": "12.1",
+      "categories": [
+        {
+          "title": "LLM Settings",
+          "settings": [
+            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
+            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
+            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
+            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
+            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
+            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
+            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
+            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
+            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
+            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
+            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
+            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
+            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
+            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
+            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
+            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
+            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
+            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
+            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
+          ]
+        },
+        {
+          "title": "Tokenizer Settings",
+          "settings": [
+            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
+          ]
+        },
+        {
+          "title": "System Settings",
+          "settings": [
+            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
+            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
+          ]
+        },
+        {
+          "title": "Streaming Settings",
+          "settings": [
+            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
+          ]
+        },
+        {
+          "title": "OpenAI Settings",
+          "settings": [
+            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
+          ]
+        },
+        {
+          "title": "Serverless Settings",
+          "settings": [
+            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
+          ]
+        }
+      ]
+    },
     "0.8.2": {
       "imageName": "runpod/worker-v1-vllm:v2.2.0stable-cuda12.1.0",
       "minimumCudaVersion": "12.1",
@@ -741,14 +985,15 @@
     "env_var_name": "QUANTIZATION",
     "value": "",
     "title": "Quantization",
-    "description": "Method used to quantize the weights.",
+    "description": "Method used to quantize the weights.\nif the `Load Format` is 'bitsandbytes' then `Quantization` will be forced to 'bitsandbytes'",
     "required": false,
     "type": "select",
     "options": [
       { "value": "None", "label": "None" },
       { "value": "awq", "label": "AWQ" },
       { "value": "squeezellm", "label": "SqueezeLLM" },
-      { "value": "gptq", "label": "GPTQ" }
+      { "value": "gptq", "label": "GPTQ" },
+      { "value": "bitsandbytes", "label": "bitsandbytes" }
     ]
   },
   "ROPE_SCALING": {