Merge pull request #304 from runpod-workers/fix/fix-tests

fix: change test gpu to L40

.github/workflows/CI-readme_version_sync.yml

@@ -0,0 +1,71 @@
+name: CI | Sync vLLM version in READMEs
+
+on:
+  push:
+    branches: ["main"]
+    paths:
+      - "Dockerfile"
+
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  sync_version:
+    runs-on: ubuntu-latest
+    name: Check README version matches Dockerfile and update if needed
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Extract vLLM version from Dockerfile and sync READMEs
+        run: |
+          echo "Extracting vLLM version from Dockerfile..."
+          dockerfile_version=$(grep -oP 'vllm(?:\[[\w,]+\])?==\K[\d.]+' Dockerfile | head -1)
+
+          if [ -z "$dockerfile_version" ]; then
+            echo "ERROR: Could not extract vLLM version from Dockerfile."
+            exit 1
+          fi
+          echo "Dockerfile vLLM version: $dockerfile_version"
+          echo "VLLM_VERSION=$dockerfile_version" >> $GITHUB_ENV
+
+          updated=0
+          for readme in README.md .runpod/README.md; do
+            if [ ! -f "$readme" ]; then
+              echo "Skipping $readme (not found)"
+              continue
+            fi
+
+            readme_version=$(grep -oP 'Current vLLM version: \[\K[\d.]+' "$readme" || echo "")
+            echo "$readme current version: ${readme_version:-not found}"
+
+            if [ "$readme_version" = "$dockerfile_version" ]; then
+              echo "$readme is already up to date."
+              continue
+            fi
+
+            echo "Updating $readme from $readme_version to $dockerfile_version..."
+            sed -i "s|Current vLLM version: \[${readme_version}\](https://github.com/vllm-project/vllm/releases/tag/v${readme_version})|Current vLLM version: [${dockerfile_version}](https://github.com/vllm-project/vllm/releases/tag/v${dockerfile_version})|g" "$readme"
+            updated=1
+          done
+
+          echo "UPDATED=$updated" >> $GITHUB_ENV
+
+      - name: Create Pull Request
+        if: env.UPDATED == '1'
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "docs: sync vLLM version to ${{ env.VLLM_VERSION }} in READMEs"
+          title: "docs: sync vLLM version to ${{ env.VLLM_VERSION }} in READMEs"
+          body: |
+            The vLLM version in the Dockerfile has been updated to `${{ env.VLLM_VERSION }}`.
+
+            This PR syncs the version badge/link in:
+            - `README.md`
+            - `.runpod/README.md`
+          branch: docs/sync-vllm-version-${{ env.VLLM_VERSION }}
+          labels: documentation

.runpod/README.md

@@ -6,7 +6,7 @@ Run LLMs using [vLLM](https://docs.vllm.ai) with an OpenAI-compatible API
 
 [![RunPod](https://api.runpod.io/badge/runpod-workers/worker-vllm)](https://www.runpod.io/console/hub/runpod-workers/worker-vllm)
 
-Current vLLM version: [0.19.1](https://github.com/vllm-project/vllm/releases/tag/v0.19.1)
+Current vLLM version: [0.22.1](https://github.com/vllm-project/vllm/releases/tag/v0.22.1)
 
 ---
 
@@ -33,6 +33,17 @@ All behaviour is controlled through environment variables:
 
 **Pass any vLLM engine arg** not listed above by setting an env var with the **UPPERCASED** field name (e.g. `MAX_MODEL_LEN=4096`, `ENABLE_CHUNKED_PREFILL=true`). The worker auto-discovers all `AsyncEngineArgs` fields from env. See the [vLLM engine args docs](https://docs.vllm.ai/en/latest/configuration/engine_args) for all available options.
 
+**Configuration file:** You can also supply a `config.yaml` instead of (or alongside) env vars. Mount it at `/vllm_config.yaml` in the container, or set `VLLM_CONFIG_FILE` to a custom path. Use the same key names as `vllm serve` — hyphens and underscores both work:
+
+```yaml
+model: meta-llama/Llama-3.1-8B-Instruct
+max-model-len: 8192
+gpu-memory-utilization: 0.90
+quantization: awq
+```
+
+Environment variables always override config file values.
+
 For complete configuration options, see the [full configuration documentation](https://github.com/runpod-workers/worker-vllm/blob/main/docs/configuration.md).
 
 ### Specify Transformers Version

.runpod/hub.json

@@ -9,7 +9,7 @@
     "containerDiskInGb": 150,
     "gpuIds": "ADA_80_PRO,AMPERE_80",
     "gpuCount": 1,
-    "allowedCudaVersions": ["12.9", "12.8"],
+    "allowedCudaVersions": ["13.0"],
     "presets": [
       {
         "name": "deepseek-ai/deepseek-r1-distill-llama-8b",
@@ -805,6 +805,16 @@
           "default": "expandable_segments:True",
           "advanced": true
         }
+      },
+      {
+        "key": "VLLM_USE_DEEP_GEMM",
+        "input": {
+          "name": "Use DeepGEMM",
+          "type": "string",
+          "description": "Enable DeepGEMM FP8 kernels (MoE and MQA logits). Set to 1 to enable, 0 to disable. Required for DeepSeek V4 models. Disabled by default — enable on H100/H200 for potential throughput gains. Some GPUs (e.g. H20) may perform better with this off.",
+          "default": "0",
+          "advanced": true
+        }
       }
     ]
   }

.runpod/tests.json

@@ -5,7 +5,7 @@
       "input": {
         "prompt": "Write a short poem about artificial intelligence."
       },
-      "timeout": 30000
+      "timeout": 300000
     },
     {
       "name": "openai_messages_test",
@@ -26,11 +26,11 @@
           "temperature": 0.1
         }
       },
-      "timeout": 30000
+      "timeout": 300000
     }
   ],
   "config": {
-    "gpuTypeId": "NVIDIA GeForce RTX 4090",
+    "gpuTypeId": "NNVIDIA L40",
     "gpuCount": 1,
     "env": [
       {
@@ -38,6 +38,6 @@
         "value": "HuggingFaceTB/SmolLM2-135M-Instruct"
       }
     ],
-    "allowedCudaVersions": ["12.9", "12.8", "12.7", "12.6", "12.5"]
+    "allowedCudaVersions": ["13.0"]
   }
 }

Dockerfile

@@ -1,16 +1,28 @@
-FROM nvidia/cuda:12.9.1-base-ubuntu22.04 
+FROM nvidia/cuda:13.0.2-devel-ubuntu22.04
 
 RUN apt-get update -y \
-    && apt-get install -y python3-pip curl \
+    && apt-get install -y python3-pip curl git \
     && curl -LsSf https://astral.sh/uv/install.sh  | sh
 
 ENV PATH="/root/.local/bin:$PATH"
 
-RUN ldconfig /usr/local/cuda-12.9/compat/
+RUN ldconfig /usr/local/cuda-13.0/compat/
 
-# Install vLLM with FlashInfer - use CUDA 12.9 PyTorch wheels
+# nixl_ep PyPI wheels are compiled against CUDA 12.x and require libcudart.so.12.
+# CUDA 13 runtime is ABI-compatible with CUDA 12, so symlinking is safe.
+# Symlink into /usr/local/cuda/lib64 (already in LD_LIBRARY_PATH) so the linker
+# finds it by filename scan rather than relying on ldcache SONAME lookup.
+RUN ln -sf /usr/local/cuda/lib64/libcudart.so.13 /usr/local/cuda/lib64/libcudart.so.12 && ldconfig
+
+# CUDA 13.0 containers return libs to /usr/local/nvidia/lib64 so container
+# providers (RunPod, Lambda, etc.) can mount host drivers there consistently.
+# See: https://github.com/vllm-project/vllm/issues/18859
+ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+
+# Install vLLM with FlashInfer - use CUDA 130 PyTorch wheels
 RUN uv pip install --system "packaging>=24.2" && \
-    uv pip install --system "vllm[flashinfer]==0.19.1" --extra-index-url https://download.pytorch.org/whl/cu129
+    uv pip install --system "vllm[flashinfer]==0.22.1" && \
+    uv pip install --system git+https://github.com/deepseek-ai/DeepGEMM.git@714dd1a4a980f7937a74343d19a8eba4fe321480 --no-build-isolation
 
 # Install additional Python dependencies (after vLLM to avoid PyTorch version conflicts)
 COPY builder/requirements.txt /requirements.txt
@@ -42,7 +54,9 @@ ENV MODEL_NAME=$MODEL_NAME \
     # Prevent rayon thread pool panic in containers where ulimit -u < nproc
     # (tokenizers uses Rust's rayon which tries to spawn threads = CPU cores)
     TOKENIZERS_PARALLELISM=false \
-    RAYON_NUM_THREADS=4
+    RAYON_NUM_THREADS=4 \
+    # Disable DeepGEMM MoE kernels by default; override with VLLM_USE_DEEP_GEMM=1 to enable
+    VLLM_USE_DEEP_GEMM=0
 
 ENV PYTHONPATH="/:/vllm-workspace"
 

README.md

@@ -8,7 +8,7 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https:
 
 ![vLLM worker banner](https://image.runpod.ai/preview/vllm/vllm-banner.png)
 
-Current vLLM version: [0.19.1](https://github.com/vllm-project/vllm/releases/tag/v0.19.1)
+Current vLLM version: [0.22.1](https://github.com/vllm-project/vllm/releases/tag/v0.22.1)
 
 
 > Check out our Load Balancer implementation here: [vLLM Load Balancer](https://github.com/runpod-workers/vllm-loadbalancer-ep)
@@ -47,7 +47,7 @@ Current vLLM version: [0.19.1](https://github.com/vllm-project/vllm/releases/tag
 **📦 Docker Image**: `runpod/worker-v1-vllm:<version>`
 
 - **Available Versions**: See [GitHub Releases](https://github.com/runpod-workers/worker-vllm/releases)
-- **CUDA Compatibility**: Requires CUDA >= 12.1
+- **CUDA Compatibility**: Requires CUDA >= 13.0
 
 ### Configuration
 
@@ -78,6 +78,20 @@ Configure worker-vllm using environment variables:
 
 Any env var whose name matches a valid `AsyncEngineArgs` field (uppercased) is applied automatically. Backward-compat aliases: `MODEL_NAME`, `TOKENIZER_NAME`, `MAX_CONTEXT_LEN_TO_CAPTURE`. This lets you configure any vLLM option without waiting for explicit worker support.
 
+### Configuration File (config.yaml)
+
+As an alternative to environment variables, you can supply a `config.yaml` file using the same key names as `vllm serve` (hyphens or underscores both work):
+
+```yaml
+model: meta-llama/Llama-3.1-8B-Instruct
+max-model-len: 8192
+gpu-memory-utilization: 0.90
+quantization: awq
+tensor-parallel-size: 2
+```
+
+Mount the file into the container at `/vllm_config.yaml`, or point to a custom path with the `VLLM_CONFIG_FILE` env var. Environment variables always take precedence over config file values.
+
 For the complete list of all available environment variables, examples, and detailed descriptions: **[Configuration](docs/configuration.md)**
 
 ### Specify Transformers Version

builder/requirements.txt

@@ -1,9 +1,9 @@
 ray
 pandas
 pyarrow
-runpod==1.9.0
+runpod==1.9.1
 huggingface-hub
-lmcache==0.4.2
+lmcache==0.4.6
 packaging>=24.2
 typing-extensions>=4.8.0
 pydantic
@@ -11,5 +11,5 @@ pydantic-settings
 hf-transfer
 transformers>=5
 bitsandbytes>=0.45.0
-kernels
+kernels<0.15
 torch-c-dlpack-ext

configs/llama/llama3.1_8b_instruct.yaml

@@ -0,0 +1,10 @@
+model: meta-llama/Llama-3.1-8B-Instruct
+gpu-memory-utilization: 0.95
+max-model-len: 8192
+dtype: auto
+trust-remote-code: true
+quantization: fp8
+kv-cache-dtype: fp8
+enforce-eager: false
+enable-prefix-caching: true
+speculative-config: '{"model":"RedHatAI/Llama-3.1-8B-Instruct-speculator.eagle3","method":"eagle3","num_speculative_tokens":3}'

configs/qwen/qwen3_8b.yaml

@@ -0,0 +1,10 @@
+model: Qwen/Qwen3-8B
+gpu-memory-utilization: 0.95
+max-model-len: 8192
+dtype: auto
+trust-remote-code: true
+quantization: fp8
+kv-cache-dtype: fp8
+enforce-eager: false
+enable-prefix-caching: true
+speculative-config: '{"model":"RedHatAI/Qwen3-8B-speculator.eagle3","method":"eagle3","num_speculative_tokens":3}'

docs/configuration.md

@@ -97,10 +97,13 @@ If `SPECULATIVE_CONFIG` is set, it takes priority over individual env vars. When
 | `MAX_SEQ_LEN_TO_CAPTURE`       | `8192`  | `int`           | Maximum context length covered by CUDA graphs. When a sequence has context length larger than this, we fall back to eager mode.     |
 | `DISABLE_CUSTOM_ALL_REDUCE`    | `0`     | `int`           | Enables or disables custom all reduce.                                                                                              |
 | `ENABLE_EXPERT_PARALLEL`       | `False` | `bool`          | Enable Expert Parallel for MoE models.                                                                                              |
+| `VLLM_USE_DEEP_GEMM`           | `0`     | `str` (`0`/`1`) | Enable DeepGEMM FP8 kernels for MoE and MQA logits computation. Disabled by default. Must be `"0"` or `"1"` — not `true`/`false`. See note below. |
 | `ATTENTION_BACKEND`            | `None`  | `str`           | Attention backend to use (e.g., `FLASH_ATTN`, `FLASHINFER`, `TRITON_FLASH_ATTN`). Replaces deprecated `VLLM_ATTENTION_BACKEND`.     |
 | `ASYNC_SCHEDULING`             | `None`  | `bool`          | Enable async scheduling (overlaps engine scheduling with GPU execution). Default: enabled in vLLM 0.14.0+. Set to `false` to disable. |
 | `STREAM_INTERVAL`              | `1`     | `int`           | Controls how often to yield streaming results. Lower = more frequent updates.                                                        |
 
+> **Note (`VLLM_USE_DEEP_GEMM`):** DeepGEMM is used in two places: MoE weight computation and MQA logits computation. It is necessary for MQA logits computation on supported hardware — required for DeepSeek V4 models. Set `VLLM_USE_DEEP_GEMM=1` to enable. Set `VLLM_USE_DEEP_GEMM=0` to disable the MoE part and fall back to flashinfer/cutlass FP8 kernels. **Value must be `"0"` or `"1"` — not `"true"`/`"false"`.** Some users report better performance with `VLLM_USE_DEEP_GEMM=0`, particularly on H20 GPUs. Disabling it also skips the DeepGEMM warmup phase, reducing cold-start time. Requires CUDA 13.0+ and SM90+ (H100/H200) to use; the library is installed but inactive by default.
+
 ## Tokenizer Settings
 
 | Variable               | Default | Type/Choices                        | Description                                                                                       |

src/engine.py

@@ -1,4 +1,5 @@
 import asyncio
+import inspect
 import json
 import logging
 import os
@@ -7,6 +8,7 @@ from typing import AsyncGenerator, Optional
 
 from dotenv import load_dotenv
 from vllm import AsyncLLMEngine
+from vllm.inputs import TextPrompt
 from vllm.entrypoints.logger import RequestLogger
 from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest, AnthropicMessagesResponse, AnthropicError, AnthropicErrorResponse
 from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
@@ -30,20 +32,33 @@ class vLLMEngine:
     def __init__(self, engine = None):
         load_dotenv() # For local development
         self.engine_args = get_engine_args()
-        logging.info(f"Engine args: {self.engine_args}")
+
-        
+        if engine is None:
-        # Initialize vLLM engine first
+            ea = self.engine_args
-        self.llm = self._initialize_llm() if engine is None else engine.llm
+            summary = {
-        
+                "model": ea.model,
-        # Only create custom tokenizer wrapper if not using mistral tokenizer mode
+                "dtype": ea.dtype,
-        # For mistral models, let vLLM handle tokenizer initialization
+                "quantization": ea.quantization,
-        if self.engine_args.tokenizer_mode != 'mistral':
+                "max_model_len": ea.max_model_len,
-            self.tokenizer = TokenizerWrapper(self.engine_args.tokenizer or self.engine_args.model, 
+                "tensor_parallel_size": ea.tensor_parallel_size,
-                                              self.engine_args.tokenizer_revision, 
+                "gpu_memory_utilization": ea.gpu_memory_utilization,
-                                              self.engine_args.trust_remote_code)
+            }
+            if ea.tokenizer and ea.tokenizer != ea.model:
+                summary["tokenizer"] = ea.tokenizer
+            logging.info("Engine config: %s", summary)
+            logging.debug("Full engine args: %s", ea)
+
+            self.llm = self._initialize_llm()
+
+            if self.engine_args.tokenizer_mode != 'mistral':
+                self.tokenizer = TokenizerWrapper(self.engine_args.tokenizer or self.engine_args.model,
+                                                  self.engine_args.tokenizer_revision,
+                                                  self.engine_args.trust_remote_code)
+            else:
+                self.tokenizer = None
         else:
-            # For mistral models, we'll get the tokenizer from vLLM later
+            self.llm = engine.llm
-            self.tokenizer = None
+            self.tokenizer = engine.tokenizer
             
         self.max_concurrency = int(os.getenv("MAX_CONCURRENCY", DEFAULT_MAX_CONCURRENCY))
         self.default_batch_size = int(os.getenv("DEFAULT_BATCH_SIZE", DEFAULT_BATCH_SIZE))
@@ -116,7 +131,7 @@ class vLLMEngine:
         if apply_chat_template or isinstance(llm_input, list):
             tokenizer_wrapper = self._get_tokenizer_for_chat_template()
             llm_input = tokenizer_wrapper.apply_chat_template(llm_input)
-        results_generator = self.llm.generate(llm_input, validated_sampling_params, request_id)
+        results_generator = self.llm.generate(TextPrompt(prompt=llm_input), validated_sampling_params, request_id)
         n_responses, n_input_tokens, is_first_output = validated_sampling_params.n, 0, True
         last_output_texts, token_counters = ["" for _ in range(n_responses)], {"batch": 0, "total": 0}
 
@@ -285,7 +300,6 @@ class OpenAIvLLMEngine(vLLMEngine):
         self.openai_serving_render = OpenAIServingRender(
             model_config=self.llm.model_config,
             renderer=self.llm.renderer,
-            io_processor=self.llm.io_processor,
             model_registry=self.serving_models.registry,
             request_logger=None,
             chat_template=chat_template,
@@ -357,8 +371,11 @@ class OpenAIvLLMEngine(vLLMEngine):
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
         )
 
-        if hasattr(self.chat_engine, 'warmup'):
+        warmup = getattr(self.chat_engine, 'warmup', None)
-            await self.chat_engine.warmup()
+        if callable(warmup):
+            result = warmup()
+            if inspect.isawaitable(result):
+                await result
 
     async def generate(self, openai_request: JobInput):
         # Ensure engines are ready (no-op if already initialized at startup)

src/engine_args.py

@@ -404,6 +404,23 @@ def _resolve_cached_model_path(model_name: str) -> str:
     return resolved
 
 
+def _get_args_from_config_file() -> dict:
+    """Load engine args from a vLLM-style config.yaml.
+
+    Checks VLLM_CONFIG_FILE env var, then falls back to /vllm_config.yaml.
+    Keys use the same long-form names as vllm serve (hyphens converted to underscores).
+    """
+    import yaml
+    path = os.getenv("VLLM_CONFIG_FILE", "/vllm_config.yaml")
+    if not os.path.exists(path):
+        return {}
+    with open(path) as f:
+        raw = yaml.safe_load(f) or {}
+    normalized = {k.replace("-", "_"): v for k, v in raw.items()}
+    logging.info("Loaded engine args from config file %s: %s", path, list(normalized.keys()))
+    return normalized
+
+
 def get_local_args():
     """
     Retrieve local arguments from a JSON file.
@@ -429,6 +446,9 @@ def get_engine_args():
     # Start with worker custom defaults (only where we differ from vLLM)
     args = dict(DEFAULT_ARGS)
 
+    # Config file values sit above defaults but below env vars
+    args.update(_get_args_from_config_file())
+
     # Auto-discover: every AsyncEngineArgs field from env UPPERCASED (e.g. MAX_MODEL_LEN)
     args.update(_get_args_from_env_auto_discover())
 

src/tokenizer.py

@@ -1,10 +1,12 @@
-from transformers import AutoTokenizer
+import logging
 import os
 from typing import Union
 
+from transformers import AutoTokenizer
+
 class TokenizerWrapper:
     def __init__(self, tokenizer_name_or_path, tokenizer_revision, trust_remote_code):
-        print(f"tokenizer_name_or_path: {tokenizer_name_or_path}, tokenizer_revision: {tokenizer_revision}, trust_remote_code: {trust_remote_code}")
+        logging.debug("tokenizer_name_or_path: %s, tokenizer_revision: %s, trust_remote_code: %s", tokenizer_name_or_path, tokenizer_revision, trust_remote_code)
         self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, revision=tokenizer_revision or "main", trust_remote_code=trust_remote_code)
         self.custom_chat_template = os.getenv("CUSTOM_CHAT_TEMPLATE")
         self.has_chat_template = bool(self.tokenizer.chat_template) or bool(self.custom_chat_template)