Merge pull request #154 from runpod-workers/fx-eng

update engine.py
2025-01-28 21:40:26 -08:00 · 2025-01-28 21:40:00 -08:00 · 2025-01-28 20:58:40 -08:00 · 2025-01-28 20:58:07 -08:00 · 2025-01-28 19:02:59 -08:00 · 2025-01-28 19:00:57 -08:00
6 changed files with 1091 additions and 918 deletions
@@ -12,7 +12,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
    python3 -m pip install --upgrade -r /requirements.txt

 # Install vLLM (switching back to pip installs since issues that required building fork are fixed and space optimization is not as important since caching) and FlashInfer 
-RUN python3 -m pip install vllm==0.6.3 && \
+RUN python3 -m pip install vllm==0.7.0 && \
    python3 -m pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3

 # Setup for Option 2: Building the Image with the Model included
@@ -32,7 +32,7 @@ ENV MODEL_NAME=$MODEL_NAME \
    HF_DATASETS_CACHE="${BASE_PATH}/huggingface-cache/datasets" \
    HUGGINGFACE_HUB_CACHE="${BASE_PATH}/huggingface-cache/hub" \
    HF_HOME="${BASE_PATH}/huggingface-cache/hub" \
-    HF_HUB_ENABLE_HF_TRANSFER=1 
+    HF_HUB_ENABLE_HF_TRANSFER=0 

 ENV PYTHONPATH="/:/vllm-workspace"

@@ -18,9 +18,9 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https:
 ### 1. UI for Deploying vLLM Worker on RunPod console:
 ![Demo of Deploying vLLM Worker on RunPod console with new UI](media/ui_demo.gif)

-### 2. Worker vLLM `v1.5.0` with vLLM `0.6.2` now available under `stable` tags 
+### 2. Worker vLLM `v1.9.0` with vLLM `0.7.0` now available under `stable` tags 

-Update v1.5.0 is now available, use the image tag `runpod/worker-v1-vllm:v1.5.0stable-cuda12.1.0`.
+Update v1.9.0 is now available, use the image tag `runpod/worker-v1-vllm:v1.9.0stable-cuda12.1.0`.

 ### 3. OpenAI-Compatible [Embedding Worker](https://github.com/runpod-workers/worker-infinity-embedding) Released
 Deploy your own OpenAI-compatible Serverless Endpoint on RunPod with multiple embedding models and fast inference for RAG and more! 
@@ -82,7 +82,7 @@ Below is a summary of the available RunPod Worker images, categorized by image s

 | CUDA Version | Stable Image Tag                  | Development Image Tag             | Note                                                        |
 |--------------|-----------------------------------|-----------------------------------|----------------------------------------------------------------------|
-| 12.1.0       | `runpod/worker-v1-vllm:v1.5.0stable-cuda12.1.0` | `runpod/worker-v1-vllm:v1.5.0dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. |
+| 12.1.0       | `runpod/worker-v1-vllm:v1.9.0stable-cuda12.1.0` | `runpod/worker-v1-vllm:v1.9.0dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. |



@@ -4,14 +4,16 @@ import json
 import asyncio

 from dotenv import load_dotenv
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Optional
 import time

 from vllm import AsyncLLMEngine
+from vllm.entrypoints.logger import RequestLogger
 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
 from vllm.entrypoints.openai.protocol import ChatCompletionRequest, CompletionRequest, ErrorResponse
-from vllm.entrypoints.openai.serving_engine import BaseModelPath
+from vllm.entrypoints.openai.serving_models import BaseModelPath, LoRAModulePath, OpenAIServingModels
+

 from utils import DummyRequest, JobInput, BatchSize, create_error_response
 from constants import DEFAULT_MAX_CONCURRENCY, DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE_GROWTH_FACTOR, DEFAULT_MIN_BATCH_SIZE
@@ -128,23 +130,44 @@ class OpenAIvLLMEngine(vLLMEngine):
        self.base_model_paths = [
            BaseModelPath(name=self.engine_args.model, model_path=self.engine_args.model)
        ]
-        self.chat_engine = OpenAIServingChat(
+
+        lora_modules = os.getenv('LORA_MODULES', None)
+        if lora_modules is not None:
+            try:
+                lora_modules = json.loads(lora_modules)
+                lora_modules = [LoRAModulePath(**lora_modules)]
+            except:
+                lora_modules = None
+
+        self.serving_models = OpenAIServingModels(
            engine_client=self.llm,
            model_config=self.model_config,
            base_model_paths=self.base_model_paths,
-            response_role=self.response_role,
-            chat_template=self.tokenizer.tokenizer.chat_template,
            lora_modules=None,
            prompt_adapters=None,
-            request_logger=None
+        )
+
+        self.chat_engine = OpenAIServingChat(
+            engine_client=self.llm, 
+            model_config=self.model_config,
+            models=self.serving_models,
+            response_role=self.response_role,
+            request_logger=None,
+            chat_template=self.tokenizer.tokenizer.chat_template,
+            chat_template_content_format="auto",
+            # enable_reasoning=os.getenv('ENABLE_REASONING', 'false').lower() == 'true',
+            # reasoning_parser=None,
+            # return_token_as_token_ids=False,
+            enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
+            tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None,
+            enable_prompt_tokens_details=False
        )
        self.completion_engine = OpenAIServingCompletion(
            engine_client=self.llm, 
            model_config=self.model_config,
-            base_model_paths=self.base_model_paths,
-            lora_modules=[],
-            prompt_adapters=None,
-            request_logger=None
+            models=self.serving_models,
+            request_logger=None,
+            # return_token_as_token_ids=False,
        )
    
    async def generate(self, openai_request: JobInput):
@@ -157,10 +180,7 @@ class OpenAIvLLMEngine(vLLMEngine):
            yield create_error_response("Invalid route").model_dump()
    
    async def _handle_model_request(self):
-        models = await self.chat_engine.show_available_models()
-        fixed_model = models.data[0]
-        fixed_model.id = self.served_model_name
-        models.data = [fixed_model]
+        models = await self.serving_models.show_available_models()
        return models.model_dump()
    
    async def _handle_chat_or_completion_request(self, openai_request: JobInput):
@@ -4,6 +4,7 @@ import logging
 from torch.cuda import device_count
 from vllm import AsyncEngineArgs
 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
+from src.utils import convert_limit_mm_per_prompt

 RENAME_ARGS_MAP = {
    "MODEL_NAME": "model",
@@ -89,7 +90,7 @@ DEFAULT_ARGS = {
    "qlora_adapter_name_or_path": os.getenv('QLORA_ADAPTER_NAME_OR_PATH', None),
    "disable_logprobs_during_spec_decoding": os.getenv('DISABLE_LOGPROBS_DURING_SPEC_DECODING', None),
    "otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None),
-    "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true')
+    "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true'),
 }

 def match_vllm_args(args):
@@ -15,6 +15,10 @@ except ImportError:

 logging.basicConfig(level=logging.INFO)

+def convert_limit_mm_per_prompt(input_string: str):
+    key, value = input_string.split('=')
+    return {key: int(value)}
+
 def count_physical_cores():
    with open('/proc/cpuinfo') as f:
        content = f.readlines()
@@ -1,5 +1,249 @@
 {
  "versions": {
+    "0.7.0": {
+      "imageName": "runpod/worker-v1-vllm:v1.9.0stable-cuda12.1.0",
+      "minimumCudaVersion": "12.1",
+      "categories": [
+        {
+          "title": "LLM Settings",
+          "settings": [
+            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
+            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
+            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
+            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
+            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
+            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
+            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
+            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
+            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
+            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
+            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
+            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
+            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
+            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
+            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
+            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
+            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
+            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
+            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
+          ]
+        },
+        {
+          "title": "Tokenizer Settings",
+          "settings": [
+            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
+          ]
+        },
+        {
+          "title": "System Settings",
+          "settings": [
+            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
+            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
+          ]
+        },
+        {
+          "title": "Streaming Settings",
+          "settings": [
+            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
+          ]
+        },
+        {
+          "title": "OpenAI Settings",
+          "settings": [
+            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
+          ]
+        },
+        {
+          "title": "Serverless Settings",
+          "settings": [
+            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
+          ]
+        }
+      ]
+    },
+    "0.6.6": {
+      "imageName": "runpod/worker-v1-vllm:v1.8.0stable-cuda12.1.0",
+      "minimumCudaVersion": "12.1",
+      "categories": [
+        {
+          "title": "LLM Settings",
+          "settings": [
+            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
+            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
+            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
+            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
+            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
+            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
+            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
+            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
+            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
+            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
+            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
+            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
+            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
+            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
+            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
+            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
+            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
+            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
+            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
+          ]
+        },
+        {
+          "title": "Tokenizer Settings",
+          "settings": [
+            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
+          ]
+        },
+        {
+          "title": "System Settings",
+          "settings": [
+            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
+            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
+          ]
+        },
+        {
+          "title": "Streaming Settings",
+          "settings": [
+            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
+          ]
+        },
+        {
+          "title": "OpenAI Settings",
+          "settings": [
+            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
+          ]
+        },
+        {
+          "title": "Serverless Settings",
+          "settings": [
+            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
+          ]
+        }
+      ]
+    },
+    "0.6.4": {
+      "imageName": "runpod/worker-v1-vllm:v1.7.0stable-cuda12.1.0",
+      "minimumCudaVersion": "12.1",
+      "categories": [
+        {
+          "title": "LLM Settings",
+          "settings": [
+            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
+            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
+            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
+            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
+            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
+            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
+            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
+            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
+            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
+            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
+            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
+            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
+            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
+            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
+            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
+            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
+            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
+            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
+            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
+          ]
+        },
+        {
+          "title": "Tokenizer Settings",
+          "settings": [
+            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
+          ]
+        },
+        {
+          "title": "System Settings",
+          "settings": [
+            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
+            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
+          ]
+        },
+        {
+          "title": "Streaming Settings",
+          "settings": [
+            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
+          ]
+        },
+        {
+          "title": "OpenAI Settings",
+          "settings": [
+            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
+          ]
+        },
+        {
+          "title": "Serverless Settings",
+          "settings": [
+            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
+          ]
+        }
+      ]
+    },
+    "0.6.3": {
+      "imageName": "runpod/worker-v1-vllm:v1.6.0stable-cuda12.1.0",
+      "minimumCudaVersion": "12.1",
+      "categories": [
+        {
+          "title": "LLM Settings",
+          "settings": [
+            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
+            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
+            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
+            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
+            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
+            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
+            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
+            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
+            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
+            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
+            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
+            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
+            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
+            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
+            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
+            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
+            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
+            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
+            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
+          ]
+        },
+        {
+          "title": "Tokenizer Settings",
+          "settings": [
+            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
+          ]
+        },
+        {
+          "title": "System Settings",
+          "settings": [
+            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
+            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
+          ]
+        },
+        {
+          "title": "Streaming Settings",
+          "settings": [
+            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
+          ]
+        },
+        {
+          "title": "OpenAI Settings",
+          "settings": [
+            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
+          ]
+        },
+        {
+          "title": "Serverless Settings",
+          "settings": [
+            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
+          ]
+        }
+      ]
+    },
    "0.6.2": {
      "imageName": "runpod/worker-v1-vllm:v1.5.0stable-cuda12.1.0",
      "minimumCudaVersion": "12.1",
@@ -119,126 +363,6 @@
          ]
        }
      ]
-      },
-      "0.5.5": {
-        "imageName": "runpod/worker-v1-vllm:v1.3.1stable-cuda12.1.0",
-        "minimumCudaVersion": "12.1",
-        "categories": [
-          {
-            "title": "LLM Settings",
-            "settings": [
-              "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
-              "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
-              "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
-              "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
-              "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
-              "DISABLE_SLIDING_WINDOW", "USE_V2_BLOCK_MANAGER", "NUM_LOOKAHEAD_SLOTS",
-              "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
-              "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
-              "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
-              "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
-              "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
-              "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
-              "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
-              "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
-              "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
-              "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
-              "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
-              "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST"
-            ]
-          },
-          {
-            "title": "Tokenizer Settings",
-            "settings": [
-              "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
-            ]
-          },
-          {
-            "title": "System Settings",
-            "settings": [
-              "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
-              "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
-            ]
-          },
-          {
-            "title": "Streaming Settings",
-            "settings": [
-              "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
-            ]
-          },
-          {
-            "title": "OpenAI Settings",
-            "settings": [
-              "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
-            ]
-          },
-          {
-            "title": "Serverless Settings",
-            "settings": [
-              "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
-            ]
-          }
-        ]
-      },
-      "0.5.4": {
-        "imageName": "runpod/worker-v1-vllm:v1.2.0stable-cuda12.1.0",
-        "minimumCudaVersion": "12.1",
-        "categories": [
-          {
-            "title": "LLM Settings",
-            "settings": [
-              "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
-              "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
-              "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
-              "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
-              "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
-              "DISABLE_SLIDING_WINDOW", "USE_V2_BLOCK_MANAGER", "NUM_LOOKAHEAD_SLOTS",
-              "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
-              "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
-              "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
-              "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
-              "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
-              "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
-              "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
-              "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
-              "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
-              "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
-              "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
-              "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST"
-            ]
-          },
-          {
-            "title": "Tokenizer Settings",
-            "settings": [
-              "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
-            ]
-          },
-          {
-            "title": "System Settings",
-            "settings": [
-              "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
-              "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
-            ]
-          },
-          {
-            "title": "Streaming Settings",
-            "settings": [
-              "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
-            ]
-          },
-          {
-            "title": "OpenAI Settings",
-            "settings": [
-              "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
-            ]
-          },
-          {
-            "title": "Serverless Settings",
-            "settings": [
-              "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
-            ]
-          }
-        ]
    }
  },
  "schema": {
@@ -931,6 +1055,30 @@
    "description": "Enables or disables vLLM request logging",
    "required": false,
    "type": "toggle"
+  },
+  "ENABLE_AUTO_TOOL_CHOICE": {
+    "env_var_name": "ENABLE_AUTO_TOOL_CHOICE",
+    "value": false,
+    "title": "Enable Auto Tool Choice",
+    "description": "Enables or disables auto tool choice",
+    "required": false,
+    "type": "toggle"
+  },
+  "TOOL_CALL_PARSER": {
+    "env_var_name": "TOOL_CALL_PARSER",
+    "value": "",
+    "title": "Tool Call Parser",
+    "description": "Tool call parser",
+    "required": false,
+    "type": "select",
+    "options": [
+      { "value": "", "label": "None" },
+      { "value": "hermes", "label": "Hermes" },
+      { "value": "mistral", "label": "Mistral" },
+      { "value": "llama3_json", "label": "Llama3 JSON" },
+      { "value": "pythonic", "label": "Pythonic" },
+      { "value": "internlm", "label": "InternLM" }
+    ]
  }
 }
 }
Author	SHA1	Message	Date
Marut PandyaandGitHub	9e8d9196b0	Merge pull request #154 from runpod-workers/fx-eng update engine.py	2025-01-28 21:40:26 -08:00
pandyamarut	30dd7c1eb5	update engine.py Signed-off-by: pandyamarut <pandyamarut@gmail.com>	2025-01-28 21:40:00 -08:00
Marut PandyaandGitHub	aa7c37ffb0	Merge pull request #152 from runpod-workers/fix0.7.0-1 update oai serving classes	2025-01-28 20:58:40 -08:00
pandyamarut	dc8c88027a	update serving classes Signed-off-by: pandyamarut <pandyamarut@gmail.com>	2025-01-28 20:58:07 -08:00
Marut PandyaandGitHub	a948e90caa	Merge pull request #151 from runpod-workers/fix0.7.0 dynamic lora loading	2025-01-28 19:02:59 -08:00
pandyamarut	c703254f71	dynamic lora loading Signed-off-by: pandyamarut <pandyamarut@gmail.com>	2025-01-28 19:00:57 -08:00
Marut PandyaandGitHub	2747106403	Merge pull request #150 from runpod-workers/up-0.7.0 update vllm 0.7.0	2025-01-28 14:27:37 -08:00
pandyamarut	91ed30e9a2	update readme Signed-off-by: pandyamarut <pandyamarut@gmail.com>	2025-01-28 14:21:33 -08:00
pandyamarut	fcbfe84f63	update vllm 0.7.0 Signed-off-by: pandyamarut <pandyamarut@gmail.com>	2025-01-28 14:18:10 -08:00
Marut PandyaandGitHub	610429df23	Merge pull request #143 from sihamouda/main Add limit_mm_per_prompt engine argument for multimodels input	2025-01-27 16:41:47 -08:00
Hailong Yang	a578c6df23	update docker file	2025-01-26 23:28:27 -05:00
sihamouda	0e3359a70e	add engine argument for multimodels input	2025-01-19 02:44:39 +01:00
pandyamarut	8e3c26be14	update worker-config Signed-off-by: pandyamarut <pandyamarut@gmail.com>	2024-12-30 17:41:10 -08:00
pandyamarut	66ea8b1110	update engine Signed-off-by: pandyamarut <pandyamarut@gmail.com>	2024-12-30 17:13:03 -08:00
Marut PandyaandGitHub	a3d432afdf	Merge pull request #140 from runpod-workers/nw-updte upgrade vllm version	2024-12-30 15:23:44 -08:00
pandyamarut	06c2bb1715	upgrade vllm version Signed-off-by: pandyamarut <pandyamarut@gmail.com>	2024-12-30 15:20:11 -08:00
Marut PandyaandGitHub	d2e355eae9	Merge pull request #137 from kldzj/fix-tool-calling fix: openai tool calling	2024-12-15 20:50:47 -08:00
kldzj	b7787d8ca8	fix: remove unused import	2024-12-06 16:29:06 +01:00
kldzj	b1fca5d257	fix: move tool flags away from engine args	2024-12-06 16:07:23 +01:00
kldzj	3e86d16892	fix: openai tool calling	2024-12-06 13:08:48 +01:00
Marut PandyaandGitHub	d9f54ce76a	Update README.md	2024-11-29 16:30:45 -08:00
Marut PandyaandGitHub	149da95cd0	Merge pull request #133 from kldzj/tool-calling feat: add tool calling flags	2024-11-29 16:29:56 -08:00
Marut PandyaandGitHub	0a89394f1d	Update worker-config.json	2024-11-29 16:29:33 -08:00
Nikolai Kolodziej	8df7f41f1d	fix: set empty `tool_call_parser` to None	2024-11-24 06:55:43 +01:00
Nikolai Kolodziej	4d7b8c03c0	feat: tool calling flags	2024-11-24 06:51:41 +01:00
Marut PandyaandGitHub	2df915a145	Merge pull request #132 from runpod-workers/wc-uo update worker-config	2024-11-20 14:54:45 -08:00
pandyamarut	aadc025849	update worker-config Signed-off-by: pandyamarut <pandyamarut@gmail.com>	2024-11-20 14:42:56 -08:00
Marut PandyaandGitHub	8b4a49073d	Merge pull request #131 from runpod-workers/up-0.6.4 [Core]Update vllm-0.6.4	2024-11-20 10:37:46 -08:00
pandyamarut	4e10641d69	update vllm Signed-off-by: pandyamarut <pandyamarut@gmail.com>	2024-11-19 12:14:04 -08:00
Marut PandyaandGitHub	6e8696c12a	Merge pull request #121 from sven-knoblauch/lora-modules add changes for lora adapter support and /v1/models endpoint	2024-10-31 15:07:02 -04:00
Marut PandyaandGitHub	b49e81a75a	Merge pull request #126 from runpod-workers/up-wc-1 update worker-config	2024-10-15 17:52:13 -07:00
pandyamarut	65932f85e1	update worker-config Signed-off-by: pandyamarut <pandyamarut@gmail.com>	2024-10-15 17:47:58 -07:00
Marut PandyaandGitHub	94840cfbbb	Update README.md	2024-10-15 17:38:31 -07:00
Sven Knoblauch	677a01e8f3	update code for case of no lora adapter	2024-10-09 14:51:33 +02:00
Sven Knoblauch	5cd12ba331	add changes for lora adapter support and /v1/models endpoint	2024-10-09 11:01:12 +02:00