Merge pull request #293 from runpod-workers/fix/update-deep-gemm-hub-value

fix: update VLLM_USE_DEEP_GEMM hub to default to 0

.github/workflows/release.yml

@@ -1,8 +1,6 @@
 name: Release
 
 on:
-  release:
-    types: [published]
   push:
     tags:
       - "v[0-9]+.[0-9]+.[0-9]+*"

.runpod/README.md

@@ -6,7 +6,7 @@ Run LLMs using [vLLM](https://docs.vllm.ai) with an OpenAI-compatible API
 
 [![RunPod](https://api.runpod.io/badge/runpod-workers/worker-vllm)](https://www.runpod.io/console/hub/runpod-workers/worker-vllm)
 
-Current vLLM version: [0.18.1](https://github.com/vllm-project/vllm/releases/tag/v0.16.0)
+Current vLLM version: [0.20.2](https://github.com/vllm-project/vllm/releases/tag/v0.20.2)
 
 ---
 
@@ -29,6 +29,7 @@ All behaviour is controlled through environment variables:
 | `REASONING_PARSER`                  | Parser for reasoning-capable models               |                     | "deepseek_r1", "qwen3", "granite", "hunyuan_a13b"                  |
 | `OPENAI_SERVED_MODEL_NAME_OVERRIDE` | Override served model name in API                 |                     | String                                                             |
 | `MAX_CONCURRENCY`                   | Maximum concurrent requests                       | 300                 | Integer                                                            |
+| `ENFORCE_EAGER`                     |  If True, we will disable CUDA graph and always execute the model in eager mode. If False, we will use CUDA graph and eager execution in hybrid for maximal performance and flexibility. | true | boolean (true or false) | 
 
 **Pass any vLLM engine arg** not listed above by setting an env var with the **UPPERCASED** field name (e.g. `MAX_MODEL_LEN=4096`, `ENABLE_CHUNKED_PREFILL=true`). The worker auto-discovers all `AsyncEngineArgs` fields from env. See the [vLLM engine args docs](https://docs.vllm.ai/en/latest/configuration/engine_args) for all available options.
 

.runpod/hub.json

@@ -9,7 +9,7 @@
     "containerDiskInGb": 150,
     "gpuIds": "ADA_80_PRO,AMPERE_80",
     "gpuCount": 1,
-    "allowedCudaVersions": ["12.9", "12.8"],
+    "allowedCudaVersions": ["13.0"],
     "presets": [
       {
         "name": "deepseek-ai/deepseek-r1-distill-llama-8b",
@@ -621,7 +621,7 @@
           "name": "Enforce Eager",
           "type": "boolean",
           "description": "Always use eager-mode PyTorch. If False (0), will use eager mode and CUDA graph in hybrid for maximal performance and flexibility",
-          "default": false,
+          "default": true,
           "advanced": true
         }
       },
@@ -795,6 +795,26 @@
           "default": "",
           "advanced": true
         }
+      },
+      {
+        "key": "PYTORCH_ALLOC_CONF",
+        "input": {
+          "name": "PyTorch Alloc Config",
+          "type": "string",
+          "description": "PyTorch allocation configuration, remove this if you want to use the default configuration",
+          "default": "expandable_segments:True",
+          "advanced": true
+        }
+      },
+      {
+        "key": "VLLM_USE_DEEP_GEMM",
+        "input": {
+          "name": "Use DeepGEMM",
+          "type": "string",
+          "description": "Enable DeepGEMM FP8 kernels (MoE and MQA logits). Set to 1 to enable, 0 to disable. Required for DeepSeek V4 models. Disabled by default — enable on H100/H200 for potential throughput gains. Some GPUs (e.g. H20) may perform better with this off.",
+          "default": "0",
+          "advanced": true
+        }
       }
     ]
   }

Dockerfile

@@ -1,16 +1,17 @@
-FROM nvidia/cuda:12.9.1-base-ubuntu22.04 
+FROM nvidia/cuda:13.0.2-devel-ubuntu22.04
 
 RUN apt-get update -y \
-    && apt-get install -y python3-pip curl \
+    && apt-get install -y python3-pip curl git \
     && curl -LsSf https://astral.sh/uv/install.sh  | sh
 
 ENV PATH="/root/.local/bin:$PATH"
 
-RUN ldconfig /usr/local/cuda-12.9/compat/
+RUN ldconfig /usr/local/cuda-13.0/compat/
 
-# Install vLLM with FlashInfer - use CUDA 12.9 PyTorch wheels
+# Install vLLM with FlashInfer - use CUDA 130 PyTorch wheels
 RUN uv pip install --system "packaging>=24.2" && \
-    uv pip install --system "vllm[flashinfer]==0.18.1" --extra-index-url https://download.pytorch.org/whl/cu129
+    uv pip install --system "vllm[flashinfer]==0.20.2" && \
+    uv pip install --system git+https://github.com/deepseek-ai/DeepGEMM.git@714dd1a4a980f7937a74343d19a8eba4fe321480 --no-build-isolation
 
 # Install additional Python dependencies (after vLLM to avoid PyTorch version conflicts)
 COPY builder/requirements.txt /requirements.txt
@@ -42,7 +43,9 @@ ENV MODEL_NAME=$MODEL_NAME \
     # Prevent rayon thread pool panic in containers where ulimit -u < nproc
     # (tokenizers uses Rust's rayon which tries to spawn threads = CPU cores)
     TOKENIZERS_PARALLELISM=false \
-    RAYON_NUM_THREADS=4
+    RAYON_NUM_THREADS=4 \
+    # Disable DeepGEMM MoE kernels by default; override with VLLM_USE_DEEP_GEMM=1 to enable
+    VLLM_USE_DEEP_GEMM=0
 
 ENV PYTHONPATH="/:/vllm-workspace"
 

README.md

@@ -8,8 +8,8 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https:
 
 ![vLLM worker banner](https://image.runpod.ai/preview/vllm/vllm-banner.png)
 
+Current vLLM version: [0.20.2](https://github.com/vllm-project/vllm/releases/tag/v0.20.2)
 
-Current vLLM version: [0.18.1](https://github.com/vllm-project/vllm/releases/tag/v0.16.0)
 
 > Check out our Load Balancer implementation here: [vLLM Load Balancer](https://github.com/runpod-workers/vllm-loadbalancer-ep)
 
@@ -47,7 +47,7 @@ Current vLLM version: [0.18.1](https://github.com/vllm-project/vllm/releases/tag
 **📦 Docker Image**: `runpod/worker-v1-vllm:<version>`
 
 - **Available Versions**: See [GitHub Releases](https://github.com/runpod-workers/worker-vllm/releases)
-- **CUDA Compatibility**: Requires CUDA >= 12.1
+- **CUDA Compatibility**: Requires CUDA >= 13.0
 
 ### Configuration
 

builder/requirements.txt

@@ -3,13 +3,13 @@ pandas
 pyarrow
 runpod==1.9.0
 huggingface-hub
-lmcache==0.4.2
+lmcache==0.4.5
 packaging>=24.2
 typing-extensions>=4.8.0
 pydantic
 pydantic-settings
 hf-transfer
-transformers>=4.57.0,<5
+transformers>=5
 bitsandbytes>=0.45.0
 kernels
 torch-c-dlpack-ext

docs/configuration.md

@@ -97,10 +97,13 @@ If `SPECULATIVE_CONFIG` is set, it takes priority over individual env vars. When
 | `MAX_SEQ_LEN_TO_CAPTURE`       | `8192`  | `int`           | Maximum context length covered by CUDA graphs. When a sequence has context length larger than this, we fall back to eager mode.     |
 | `DISABLE_CUSTOM_ALL_REDUCE`    | `0`     | `int`           | Enables or disables custom all reduce.                                                                                              |
 | `ENABLE_EXPERT_PARALLEL`       | `False` | `bool`          | Enable Expert Parallel for MoE models.                                                                                              |
+| `VLLM_USE_DEEP_GEMM`           | `0`     | `str` (`0`/`1`) | Enable DeepGEMM FP8 kernels for MoE and MQA logits computation. Disabled by default. Must be `"0"` or `"1"` — not `true`/`false`. See note below. |
 | `ATTENTION_BACKEND`            | `None`  | `str`           | Attention backend to use (e.g., `FLASH_ATTN`, `FLASHINFER`, `TRITON_FLASH_ATTN`). Replaces deprecated `VLLM_ATTENTION_BACKEND`.     |
 | `ASYNC_SCHEDULING`             | `None`  | `bool`          | Enable async scheduling (overlaps engine scheduling with GPU execution). Default: enabled in vLLM 0.14.0+. Set to `false` to disable. |
 | `STREAM_INTERVAL`              | `1`     | `int`           | Controls how often to yield streaming results. Lower = more frequent updates.                                                        |
 
+> **Note (`VLLM_USE_DEEP_GEMM`):** DeepGEMM is used in two places: MoE weight computation and MQA logits computation. It is necessary for MQA logits computation on supported hardware — required for DeepSeek V4 models. Set `VLLM_USE_DEEP_GEMM=1` to enable. Set `VLLM_USE_DEEP_GEMM=0` to disable the MoE part and fall back to flashinfer/cutlass FP8 kernels. **Value must be `"0"` or `"1"` — not `"true"`/`"false"`.** Some users report better performance with `VLLM_USE_DEEP_GEMM=0`, particularly on H20 GPUs. Disabling it also skips the DeepGEMM warmup phase, reducing cold-start time. Requires CUDA 13.0+ and SM90+ (H100/H200) to use; the library is installed but inactive by default.
+
 ## Tokenizer Settings
 
 | Variable               | Default | Type/Choices                        | Description                                                                                       |

src/engine.py

@@ -1,4 +1,5 @@
 import asyncio
+import inspect
 import json
 import logging
 import os
@@ -7,6 +8,7 @@ from typing import AsyncGenerator, Optional
 
 from dotenv import load_dotenv
 from vllm import AsyncLLMEngine
+from vllm.inputs import TextPrompt
 from vllm.entrypoints.logger import RequestLogger
 from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest, AnthropicMessagesResponse, AnthropicError, AnthropicErrorResponse
 from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
@@ -19,6 +21,7 @@ from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePat
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
 from vllm.entrypoints.openai.responses.protocol import ResponsesRequest, ResponsesResponse
 from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
+from vllm.entrypoints.serve.render.serving import OpenAIServingRender
 
 from constants import DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE_GROWTH_FACTOR, DEFAULT_MAX_CONCURRENCY, DEFAULT_MIN_BATCH_SIZE
 from engine_args import get_engine_args
@@ -29,20 +32,33 @@ class vLLMEngine:
     def __init__(self, engine = None):
         load_dotenv() # For local development
         self.engine_args = get_engine_args()
-        logging.info(f"Engine args: {self.engine_args}")
-        
-        # Initialize vLLM engine first
-        self.llm = self._initialize_llm() if engine is None else engine.llm
-        
-        # Only create custom tokenizer wrapper if not using mistral tokenizer mode
-        # For mistral models, let vLLM handle tokenizer initialization
-        if self.engine_args.tokenizer_mode != 'mistral':
-            self.tokenizer = TokenizerWrapper(self.engine_args.tokenizer or self.engine_args.model, 
-                                              self.engine_args.tokenizer_revision, 
-                                              self.engine_args.trust_remote_code)
+
+        if engine is None:
+            ea = self.engine_args
+            summary = {
+                "model": ea.model,
+                "dtype": ea.dtype,
+                "quantization": ea.quantization,
+                "max_model_len": ea.max_model_len,
+                "tensor_parallel_size": ea.tensor_parallel_size,
+                "gpu_memory_utilization": ea.gpu_memory_utilization,
+            }
+            if ea.tokenizer and ea.tokenizer != ea.model:
+                summary["tokenizer"] = ea.tokenizer
+            logging.info("Engine config: %s", summary)
+            logging.debug("Full engine args: %s", ea)
+
+            self.llm = self._initialize_llm()
+
+            if self.engine_args.tokenizer_mode != 'mistral':
+                self.tokenizer = TokenizerWrapper(self.engine_args.tokenizer or self.engine_args.model,
+                                                  self.engine_args.tokenizer_revision,
+                                                  self.engine_args.trust_remote_code)
+            else:
+                self.tokenizer = None
         else:
-            # For mistral models, we'll get the tokenizer from vLLM later
-            self.tokenizer = None
+            self.llm = engine.llm
+            self.tokenizer = engine.tokenizer
             
         self.max_concurrency = int(os.getenv("MAX_CONCURRENCY", DEFAULT_MAX_CONCURRENCY))
         self.default_batch_size = int(os.getenv("DEFAULT_BATCH_SIZE", DEFAULT_BATCH_SIZE))
@@ -115,7 +131,7 @@ class vLLMEngine:
         if apply_chat_template or isinstance(llm_input, list):
             tokenizer_wrapper = self._get_tokenizer_for_chat_template()
             llm_input = tokenizer_wrapper.apply_chat_template(llm_input)
-        results_generator = self.llm.generate(llm_input, validated_sampling_params, request_id)
+        results_generator = self.llm.generate(TextPrompt(prompt=llm_input), validated_sampling_params, request_id)
         n_responses, n_input_tokens, is_first_output = validated_sampling_params.n, 0, True
         last_output_texts, token_counters = ["" for _ in range(n_responses)], {"batch": 0, "total": 0}
 
@@ -205,19 +221,48 @@ class OpenAIvLLMEngine(vLLMEngine):
             self.raw_openai_output = bool(int(raw_output_env))
 
     def _load_lora_adapters(self):
-        adapters = []
-        try:
-            adapters = json.loads(os.getenv("LORA_MODULES", '[]'))
-        except Exception as e:
-            logging.info(f"---Initialized adapter json load error: {e}")
+        lora_modules_env = os.getenv("LORA_MODULES", "")
+        if not lora_modules_env:
+            return []
 
-        for i, adapter in enumerate(adapters):
+        try:
+            parsed = json.loads(lora_modules_env)
+        except json.JSONDecodeError as e:
+            logging.error(
+                "LORA_MODULES could not be parsed as JSON: %s — no LoRA adapters loaded. Value: %r",
+                e, lora_modules_env,
+            )
+            return []
+
+        # Accept a single adapter dict as well as an array
+        if isinstance(parsed, dict):
+            parsed = [parsed]
+
+        if not isinstance(parsed, list):
+            logging.error(
+                "LORA_MODULES must be a JSON array of adapter objects, got %s — no LoRA adapters loaded.",
+                type(parsed).__name__,
+            )
+            return []
+
+        adapters = []
+        for i, adapter in enumerate(parsed):
             try:
-                adapters[i] = LoRAModulePath(**adapter)
-                logging.info(f"---Initialized adapter: {adapter}")
+                adapters.append(LoRAModulePath(**adapter))
+                logging.info("Loaded LoRA adapter config [%d]: %s", i, adapter)
             except Exception as e:
-                logging.info(f"---Initialized adapter not worked: {e}")
-                continue
+                logging.error(
+                    "Failed to parse LoRA adapter at index %d: %s. Config: %r",
+                    i, e, adapter,
+                )
+
+        if parsed and not adapters:
+            logging.error(
+                "LORA_MODULES specified %d adapter(s) but none could be loaded — "
+                "OpenAI model name lookups for LoRA adapters will fail.",
+                len(parsed),
+            )
+
         return adapters
 
     async def _ensure_engines_initialized(self):
@@ -246,16 +291,32 @@ class OpenAIvLLMEngine(vLLMEngine):
             lora_modules=self.lora_adapters,
         )
         await self.serving_models.init_static_loras()
-        
+
         # Get chat template from vLLM tokenizer if available
         chat_template = None
         if self.tokenizer and hasattr(self.tokenizer, 'tokenizer'):
             chat_template = self.tokenizer.tokenizer.chat_template
-        
+
+        self.openai_serving_render = OpenAIServingRender(
+            model_config=self.llm.model_config,
+            renderer=self.llm.renderer,
+            model_registry=self.serving_models.registry,
+            request_logger=None,
+            chat_template=chat_template,
+            chat_template_content_format="auto",
+            trust_request_chat_template=os.getenv('TRUST_REQUEST_CHAT_TEMPLATE', 'false').lower() == 'true',
+            enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
+            exclude_tools_when_tool_choice_none=os.getenv('EXCLUDE_TOOLS_WHEN_TOOL_CHOICE_NONE', 'false').lower() == 'true',
+            tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None,
+            reasoning_parser=os.getenv('REASONING_PARSER', "") or None,
+            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
+        )
+
         self.chat_engine = OpenAIServingChat(
-            engine_client=self.llm, 
+            engine_client=self.llm,
             models=self.serving_models,
             response_role=self.response_role,
+            openai_serving_render=self.openai_serving_render,
             request_logger=None,
             chat_template=chat_template,
             chat_template_content_format="auto",
@@ -268,20 +329,20 @@ class OpenAIvLLMEngine(vLLMEngine):
             enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
             enable_log_outputs=os.getenv('ENABLE_LOG_OUTPUTS', 'false').lower() == 'true',
-            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
         )
         self.completion_engine = OpenAIServingCompletion(
             engine_client=self.llm,
             models=self.serving_models,
+            openai_serving_render=self.openai_serving_render,
             request_logger=None,
             return_tokens_as_token_ids=os.getenv('RETURN_TOKENS_AS_TOKEN_IDS', 'false').lower() == 'true',
             enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
-            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
         )
         self.responses_engine = OpenAIServingResponses(
             engine_client=self.llm,
             models=self.serving_models,
+            openai_serving_render=self.openai_serving_render,
             request_logger=None,
             chat_template=chat_template,
             chat_template_content_format="auto",
@@ -293,12 +354,12 @@ class OpenAIvLLMEngine(vLLMEngine):
             enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
             enable_log_outputs=os.getenv('ENABLE_LOG_OUTPUTS', 'false').lower() == 'true',
-            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
         )
         self.messages_engine = AnthropicServingMessages(
             engine_client=self.llm,
             models=self.serving_models,
             response_role=self.response_role,
+            openai_serving_render=self.openai_serving_render,
             request_logger=None,
             chat_template=chat_template,
             chat_template_content_format="auto",
@@ -310,8 +371,11 @@ class OpenAIvLLMEngine(vLLMEngine):
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
         )
 
-        if hasattr(self.chat_engine, 'warmup'):
-            await self.chat_engine.warmup()
+        warmup = getattr(self.chat_engine, 'warmup', None)
+        if callable(warmup):
+            result = warmup()
+            if inspect.isawaitable(result):
+                await result
 
     async def generate(self, openai_request: JobInput):
         # Ensure engines are ready (no-op if already initialized at startup)

src/engine_args.py

@@ -82,6 +82,16 @@ def _convert_env_value_to_field_type(value: str, field_name: str, field_type: ty
         if type(None) in (args or ()):
             return None
         raise ValueError("empty value not allowed for non-optional field")
+
+    # Union[bool, str, ...]: only coerce to bool for unambiguous literals;
+    # otherwise preserve the string (e.g. hf_token="hf_abc..." must stay a str).
+    if get_origin(field_type) is not None:
+        union_types = [a for a in (get_args(field_type) or ()) if a is not type(None)]
+        if bool in union_types and str in union_types:
+            if str(val).lower() in ("true", "false", "1", "0", "yes", "no", "on", "off"):
+                return str(val).lower() in ("true", "1", "yes", "on")
+            return str(val)
+
     effective_type = _resolve_field_type(field_type)
     # bool
     if effective_type is bool:
@@ -342,6 +352,58 @@ def _sanitize_hf_overrides(hf_overrides: dict) -> dict | None:
     return result or None
 
 
+def _resolve_cached_model_path(model_name: str) -> str:
+    """Return a local snapshot path when the HF cache was stored with lowercase names.
+
+    Some model stores (e.g. RunPod pre-cached volumes) normalize repo IDs to
+    lowercase. HuggingFace Hub stores caches as
+    ``models--{org}--{model}/snapshots/{hash}/`` preserving the original casing,
+    so MODEL_NAME=Qwen/Qwen2.5-Coder-32B-Instruct-AWQ will miss a cache stored
+    as ``models--qwen--qwen2.5-coder-32b-instruct-awq/``.
+
+    If the exact-case cache directory is absent but a lowercase variant exists,
+    the latest snapshot path is returned so vLLM loads from disk rather than
+    attempting a redundant download.
+    """
+    if os.path.isabs(model_name):
+        return model_name
+
+    cache_dir = (
+        os.getenv("HUGGINGFACE_HUB_CACHE")
+        or os.getenv("HF_HOME")
+        or os.path.expanduser("~/.cache/huggingface/hub")
+    )
+
+    folder_name = f"models--{model_name.replace('/', '--')}"
+
+    if os.path.isdir(os.path.join(cache_dir, folder_name)):
+        return model_name
+
+    lower_dir = os.path.join(cache_dir, folder_name.lower())
+    if not os.path.isdir(lower_dir):
+        return model_name
+
+    snapshots_dir = os.path.join(lower_dir, "snapshots")
+    if not os.path.isdir(snapshots_dir):
+        return model_name
+
+    try:
+        snapshots = sorted(os.listdir(snapshots_dir))
+    except OSError:
+        return model_name
+
+    if not snapshots:
+        return model_name
+
+    resolved = os.path.join(snapshots_dir, snapshots[-1])
+    logging.info(
+        "MODEL_NAME %r not found at original casing in HF cache; "
+        "resolved to lowercase cached snapshot at %r",
+        model_name, resolved,
+    )
+    return resolved
+
+
 def get_local_args():
     """
     Retrieve local arguments from a JSON file.
@@ -517,4 +579,8 @@ def get_engine_args():
     if speculative_config:
         args["speculative_config"] = speculative_config
 
+    # Resolve lowercase HF cache paths (FDE-174)
+    if args.get("model"):
+        args["model"] = _resolve_cached_model_path(args["model"])
+
     return AsyncEngineArgs(**args)

src/tokenizer.py

@@ -1,10 +1,12 @@
-from transformers import AutoTokenizer
+import logging
 import os
 from typing import Union
 
+from transformers import AutoTokenizer
+
 class TokenizerWrapper:
     def __init__(self, tokenizer_name_or_path, tokenizer_revision, trust_remote_code):
-        print(f"tokenizer_name_or_path: {tokenizer_name_or_path}, tokenizer_revision: {tokenizer_revision}, trust_remote_code: {trust_remote_code}")
+        logging.debug("tokenizer_name_or_path: %s, tokenizer_revision: %s, trust_remote_code: %s", tokenizer_name_or_path, tokenizer_revision, trust_remote_code)
         self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, revision=tokenizer_revision or "main", trust_remote_code=trust_remote_code)
         self.custom_chat_template = os.getenv("CUSTOM_CHAT_TEMPLATE")
         self.has_chat_template = bool(self.tokenizer.chat_template) or bool(self.custom_chat_template)