From e6950bdebdc81a03eaf877cabba9076e725ed3fe Mon Sep 17 00:00:00 2001 From: velaraptor-runpod Date: Thu, 30 Apr 2026 17:41:11 -0500 Subject: [PATCH] feat: upgrade vLLM to 0.19.1 - Bump vllm[flashinfer] to 0.19.1 in Dockerfile - Add OpenAIServingRender (new required dependency in 0.19.x serving layer) - Pass openai_serving_render to all four serving class constructors - Remove log_error_stack param (removed upstream in 0.19.x) Co-Authored-By: Claude Sonnet 4.6 --- Dockerfile | 2 +- src/engine.py | 30 ++++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 332bf48..1f05b31 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN ldconfig /usr/local/cuda-12.9/compat/ # Install vLLM with FlashInfer - use CUDA 12.9 PyTorch wheels RUN uv pip install --system "packaging>=24.2" && \ - uv pip install --system "vllm[flashinfer]==0.18.1" --extra-index-url https://download.pytorch.org/whl/cu129 + uv pip install --system "vllm[flashinfer]==0.19.1" --extra-index-url https://download.pytorch.org/whl/cu129 # Install additional Python dependencies (after vLLM to avoid PyTorch version conflicts) COPY builder/requirements.txt /requirements.txt diff --git a/src/engine.py b/src/engine.py index 8c99326..dcbf67a 100644 --- a/src/engine.py +++ b/src/engine.py @@ -19,6 +19,7 @@ from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePat from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.responses.protocol import ResponsesRequest, ResponsesResponse from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses +from vllm.entrypoints.serve.render.serving import OpenAIServingRender from constants import DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE_GROWTH_FACTOR, DEFAULT_MAX_CONCURRENCY, DEFAULT_MIN_BATCH_SIZE from engine_args import get_engine_args @@ -246,16 +247,33 @@ class OpenAIvLLMEngine(vLLMEngine): lora_modules=self.lora_adapters, ) await self.serving_models.init_static_loras() - + # Get chat template from vLLM tokenizer if available chat_template = None if self.tokenizer and hasattr(self.tokenizer, 'tokenizer'): chat_template = self.tokenizer.tokenizer.chat_template - + + self.openai_serving_render = OpenAIServingRender( + model_config=self.llm.model_config, + renderer=self.llm.renderer, + io_processor=self.llm.io_processor, + model_registry=self.serving_models.registry, + request_logger=None, + chat_template=chat_template, + chat_template_content_format="auto", + trust_request_chat_template=os.getenv('TRUST_REQUEST_CHAT_TEMPLATE', 'false').lower() == 'true', + enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true', + exclude_tools_when_tool_choice_none=os.getenv('EXCLUDE_TOOLS_WHEN_TOOL_CHOICE_NONE', 'false').lower() == 'true', + tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None, + reasoning_parser=os.getenv('REASONING_PARSER', "") or None, + log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true', + ) + self.chat_engine = OpenAIServingChat( - engine_client=self.llm, + engine_client=self.llm, models=self.serving_models, response_role=self.response_role, + openai_serving_render=self.openai_serving_render, request_logger=None, chat_template=chat_template, chat_template_content_format="auto", @@ -268,20 +286,20 @@ class OpenAIvLLMEngine(vLLMEngine): enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true', enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true', enable_log_outputs=os.getenv('ENABLE_LOG_OUTPUTS', 'false').lower() == 'true', - log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true', ) self.completion_engine = OpenAIServingCompletion( engine_client=self.llm, models=self.serving_models, + openai_serving_render=self.openai_serving_render, request_logger=None, return_tokens_as_token_ids=os.getenv('RETURN_TOKENS_AS_TOKEN_IDS', 'false').lower() == 'true', enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true', enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true', - log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true', ) self.responses_engine = OpenAIServingResponses( engine_client=self.llm, models=self.serving_models, + openai_serving_render=self.openai_serving_render, request_logger=None, chat_template=chat_template, chat_template_content_format="auto", @@ -293,12 +311,12 @@ class OpenAIvLLMEngine(vLLMEngine): enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true', enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true', enable_log_outputs=os.getenv('ENABLE_LOG_OUTPUTS', 'false').lower() == 'true', - log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true', ) self.messages_engine = AnthropicServingMessages( engine_client=self.llm, models=self.serving_models, response_role=self.response_role, + openai_serving_render=self.openai_serving_render, request_logger=None, chat_template=chat_template, chat_template_content_format="auto",