fix: make .runpod/tests.json hub tests pass on CUDA 13.0 (#299)

Three coupled fixes verified end-to-end on a private fork
(TimPietruskyRunPod/worker-vllm v0.1.3 → both hub tests passing):

1. tests.json allowedCudaVersions: 12.x → 13.0
   The Dockerfile and hub.json moved to CUDA 13.0 in v2.20.0
   (#288, #289), but tests.json was still pinned to 12.5–12.9, so
   the test pod was scheduled on a GPU with driver < 13.0 and
   container init failed at the nvidia-container-cli hook with
   "unsatisfied condition: cuda>=13.0".

2. requirements.txt kernels<0.15
   huggingface/kernels v0.15.1 tightened LayerRepository to require
   a revision or version argument
   (https://github.com/huggingface/kernels/pull/544). transformers
   >=5 still constructs LayerRepository(repo_id=..., layer_name=...)
   without either, so worker import raised ValueError during
   `from transformers import ...`. 0.14.1 is the last safe release.

3. tests.json timeout 30000 → 300000
   vLLM cold start (torch.compile + FlashInfer warmup) on RTX 4090
   for SmolLM2-135M takes ~60–70s before the first request can be
   served. The previous 30s per-test timeout fired before the
   worker came up, producing "context cancelled or timed out:
   context deadline exceeded" for every test even when the worker
   was healthy. 300s gives enough headroom for cold start + the
   actual inference call.

Refs: DR-1161

.github/workflows/CI-runpod_dep.yml

@@ -9,59 +9,60 @@ on:
 
   workflow_dispatch:
 
+permissions:
+  contents: write
+  pull-requests: write
+
 jobs:
   check_dep:
     runs-on: ubuntu-latest
     name: Check python requirements file and update
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Check for new package version and update
         run: |
-          echo "Fetching the current runpod version from requirements.txt..."
-          
-          # Get current version, allowing both == and ~= in the search pattern
-          current_version=$(grep -oP 'runpod[~=]{1,2}\K[^"]+' ./builder/requirements.txt)
-          echo "Current version: $current_version"
+          echo "Fetching current runpod version from requirements.txt..."
 
-          # Extract major and minor from current version
-          current_major_minor=$(echo $current_version | cut -d. -f1,2)
-          echo "Current major.minor: $current_major_minor"
+          # Match runpod with any version specifier or no specifier at all
+          current_version=$(grep -oP '^runpod([~>=!<]{1,2}\K[\d.]+)?' ./builder/requirements.txt | grep -oP '[\d.]+' || echo "")
+          echo "Current version: ${current_version:-unset}"
 
-          echo "Fetching the latest runpod version from PyPI..."
-          
-          # Get new version from PyPI
-          new_version=$(curl -s https://pypi.org/pypi/runpod/json | jq -r .info.version)
+          echo "Fetching latest runpod version from PyPI..."
+          new_version=$(curl -sf https://pypi.org/pypi/runpod/json | jq -r .info.version)
           echo "NEW_VERSION_ENV=$new_version" >> $GITHUB_ENV
           echo "New version: $new_version"
 
-          # Extract major and minor from new version
-          new_major_minor=$(echo $new_version | cut -d. -f1,2)
-          echo "New major.minor: $new_major_minor"
-
           if [ -z "$new_version" ]; then
-              echo "ERROR: Failed to fetch the new version from PyPI."
-              exit 1
+            echo "ERROR: Failed to fetch new version from PyPI."
+            exit 1
           fi
 
-          # Check if the major or minor version is different
-          if [ "$current_major_minor" = "$new_major_minor" ]; then
-              echo "No update needed. The new version ($new_major_minor) is within the allowed range (~= $current_major_minor)."
+          if [ -z "$current_version" ]; then
+            echo "No version pin found — pinning to $new_version."
+          else
+            current_major_minor=$(echo "$current_version" | cut -d. -f1,2)
+            new_major_minor=$(echo "$new_version" | cut -d. -f1,2)
+            echo "Current major.minor: $current_major_minor  New major.minor: $new_major_minor"
+
+            if [ "$current_major_minor" = "$new_major_minor" ]; then
+              echo "No update needed. New version ($new_version) is within ~= $current_major_minor range."
               exit 0
+            fi
+
+            echo "New major/minor detected ($new_major_minor). Updating requirements.txt..."
           fi
 
-          echo "New major/minor detected ($new_major_minor). Updating requirements.txt..."
-
-          # Update requirements.txt, preserving the existing constraint type (~= or ==)
-          sed -i "s/runpod[~=][^ ]*/runpod~=$new_version/" ./builder/requirements.txt
-          echo "requirements.txt has been updated."
+          # Replace any `runpod`, `runpod==x`, `runpod~=x`, etc. with pinned version
+          sed -i "s|^runpod.*|runpod~=$new_version|" ./builder/requirements.txt
+          echo "requirements.txt updated."
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v3
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: Update runpod package version
-          title: Update runpod package version
-          body: The package version has been updated to ${{ env.NEW_VERSION_ENV }}
+          commit-message: "chore: update runpod to ${{ env.NEW_VERSION_ENV }}"
+          title: "chore: update runpod to ${{ env.NEW_VERSION_ENV }}"
+          body: The `runpod` package has been updated to `${{ env.NEW_VERSION_ENV }}`.
           branch: runpod-package-update

.github/workflows/release.yml

@@ -3,7 +3,7 @@ name: Release
 on:
   push:
     tags:
-      - "v[0-9]+.[0-9]+.[0-9]+*" # Trigger on version tags like v1.0.0, v2.1.0, etc.
+      - "v[0-9]+.[0-9]+.[0-9]+*"
   workflow_dispatch:
     inputs:
       version:
@@ -53,16 +53,13 @@ jobs:
 
           # Determine version based on trigger type
           if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
-            # Manual trigger: use input version
             VERSION="${{ github.event.inputs.version }}"
-            echo "RELEASE_VERSION=${VERSION}" >> $GITHUB_ENV
-            echo "IS_MANUAL_RELEASE=true" >> $GITHUB_ENV
+          elif [[ "${{ github.event_name }}" == "release" ]]; then
+            VERSION="${{ github.event.release.tag_name }}"
           else
-            # Tag trigger: use tag name (remove refs/tags/ prefix)
             VERSION=${GITHUB_REF#refs/tags/}
-            echo "RELEASE_VERSION=${VERSION}" >> $GITHUB_ENV
-            echo "IS_MANUAL_RELEASE=false" >> $GITHUB_ENV
           fi
+          echo "RELEASE_VERSION=${VERSION}" >> $GITHUB_ENV
 
       - name: Build and push the images to Docker Hub
         uses: docker/bake-action@v2
@@ -76,11 +73,45 @@ jobs:
 
       - name: Release Summary
         run: |
-          echo "🚀 Release completed!"
+          echo "Release completed!"
           echo "Version: ${{ env.RELEASE_VERSION }}"
           echo "Docker Image: ${{ env.DOCKERHUB_REPO }}/${{ env.DOCKERHUB_IMG }}:${{ env.RELEASE_VERSION }}"
-          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
-            echo "Trigger: Manual workflow dispatch"
-          else
-            echo "Trigger: GitHub release (tag: ${{ github.ref_name }})"
+
+      - name: Fetch Release Notes
+        run: |
+          RESPONSE=$(curl -sf \
+            -H "Authorization: token ${{ github.token }}" \
+            "https://api.github.com/repos/${{ github.repository }}/releases/tags/${{ env.RELEASE_VERSION }}" 2>/dev/null) || true
+          if [[ -n "$RESPONSE" ]]; then
+            NOTES=$(echo "$RESPONSE" | jq -r '.body // empty')
           fi
+          printf '%s' "${NOTES:-No release notes available.}" > /tmp/release_notes.txt
+
+      - name: Notify Slack
+        run: |
+          jq -n \
+            --arg version "${{ env.RELEASE_VERSION }}" \
+            --arg docker "${{ env.DOCKERHUB_REPO }}/${{ env.DOCKERHUB_IMG }}:${{ env.RELEASE_VERSION }}" \
+            --rawfile notes /tmp/release_notes.txt \
+            --arg url "https://github.com/${{ github.repository }}/releases/tag/${{ env.RELEASE_VERSION }}" \
+            '{
+              text: (":rocket: New :runpod-new-whiteonpurple: Runpod worker-vllm release: *" + $version + "*"),
+              blocks: [
+                {
+                  type: "section",
+                  text: {
+                    type: "mrkdwn",
+                    text: (":banana-dance: *New Release — worker-vllm " + $version + "*\n*Docker:* `" + $docker + "`\n<" + $url + "|View release on GitHub>")
+                  }
+                },
+                {
+                  type: "section",
+                  text: {
+                    type: "mrkdwn",
+                    text: ("*Release Notes:*\n" + $notes)
+                  }
+                }
+              ]
+            }' | curl -sf -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \
+              -H "Content-Type: application/json" \
+              -d @-

.github/workflows/slack-pr-issue-notify.yml

@@ -0,0 +1,41 @@
+name: Slack PR Notifications
+
+on:
+  pull_request:
+    types: [opened]
+  issues:
+    types: [opened]
+
+permissions:
+    contents: read
+   
+jobs:
+  notify:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Slack - New PR
+        if: github.event_name == 'pull_request'
+        run: |
+          curl -sf -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \
+            -H "Content-Type: application/json" \
+            -d '{
+              "text": ":rocket: New PR in worker-vllm: *${{ github.event.pull_request.title }}*",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": ":rocket: *New Pull Request — worker-vllm*\n*<${{ github.event.pull_request.html_url }}|${{ github.event.pull_request.title }}>*\nOpened by *${{ github.event.pull_request.user.login }}*"
+                  }
+                },
+                {
+                  "type": "context",
+                  "elements": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "${{ github.event.pull_request.base.ref }} ← ${{ github.event.pull_request.head.ref }}"
+                    }
+                  ]
+                }
+              ]
+            }'

.github/workflows/slack-vllm-monitor.yml

@@ -0,0 +1,73 @@
+name: Monitor vLLM Releases
+
+on:
+  schedule:
+    - cron: '0 0 * * *' # Every day at midnight
+  workflow_dispatch:
+
+
+permissions:
+    contents: read
+   
+jobs:
+  check-vllm-release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Restore last known vLLM tag
+        uses: actions/cache/restore@v4
+        with:
+          path: .vllm-last-tag
+          key: vllm-tag-${{ github.run_id }}
+          restore-keys: vllm-tag-
+
+      - name: Get latest vLLM release
+        id: vllm
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          response=$(curl -sf https://api.github.com/repos/vllm-project/vllm/releases/latest \
+            -H "Authorization: Bearer $GH_TOKEN")
+          echo "tag=$(echo "$response" | jq -r '.tag_name')" >> $GITHUB_OUTPUT
+          echo "url=$(echo "$response" | jq -r '.html_url')" >> $GITHUB_OUTPUT
+          echo "name=$(echo "$response" | jq -r '.name')" >> $GITHUB_OUTPUT
+
+      - name: Check if new release
+        id: check
+        run: |
+          last=$(cat .vllm-last-tag 2>/dev/null || echo "")
+          current="${{ steps.vllm.outputs.tag }}"
+          echo "Last: $last  Current: $current"
+          if [ -n "$current" ] && [ "$last" != "$current" ]; then
+            echo "is_new=true" >> $GITHUB_OUTPUT
+          else
+            echo "is_new=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Notify Slack
+        if: steps.check.outputs.is_new == 'true'
+        run: |
+          curl -sf -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \
+            -H "Content-Type: application/json" \
+            -d '{
+              "text": ":rocket: New vLLM release: *${{ steps.vllm.outputs.tag }}*",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": ":rocket: *New vLLM Release: ${{ steps.vllm.outputs.tag }}*\n<${{ steps.vllm.outputs.url }}|View on GitHub>"
+                  }
+                }
+              ]
+            }'
+
+      - name: Save new tag
+        if: steps.check.outputs.is_new == 'true'
+        run: echo "${{ steps.vllm.outputs.tag }}" > .vllm-last-tag
+
+      - name: Update cache
+        if: steps.check.outputs.is_new == 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: .vllm-last-tag
+          key: vllm-tag-${{ steps.vllm.outputs.tag }}

.runpod/README.md

@@ -6,6 +6,8 @@ Run LLMs using [vLLM](https://docs.vllm.ai) with an OpenAI-compatible API
 
 [![RunPod](https://api.runpod.io/badge/runpod-workers/worker-vllm)](https://www.runpod.io/console/hub/runpod-workers/worker-vllm)
 
+Current vLLM version: [0.20.2](https://github.com/vllm-project/vllm/releases/tag/v0.20.2)
+
 ---
 
 ## Endpoint Configuration
@@ -27,6 +29,7 @@ All behaviour is controlled through environment variables:
 | `REASONING_PARSER`                  | Parser for reasoning-capable models               |                     | "deepseek_r1", "qwen3", "granite", "hunyuan_a13b"                  |
 | `OPENAI_SERVED_MODEL_NAME_OVERRIDE` | Override served model name in API                 |                     | String                                                             |
 | `MAX_CONCURRENCY`                   | Maximum concurrent requests                       | 300                 | Integer                                                            |
+| `ENFORCE_EAGER`                     |  If True, we will disable CUDA graph and always execute the model in eager mode. If False, we will use CUDA graph and eager execution in hybrid for maximal performance and flexibility. | true | boolean (true or false) | 
 
 **Pass any vLLM engine arg** not listed above by setting an env var with the **UPPERCASED** field name (e.g. `MAX_MODEL_LEN=4096`, `ENABLE_CHUNKED_PREFILL=true`). The worker auto-discovers all `AsyncEngineArgs` fields from env. See the [vLLM engine args docs](https://docs.vllm.ai/en/latest/configuration/engine_args) for all available options.
 

.runpod/hub.json

@@ -9,7 +9,7 @@
     "containerDiskInGb": 150,
     "gpuIds": "ADA_80_PRO,AMPERE_80",
     "gpuCount": 1,
-    "allowedCudaVersions": ["12.9", "12.8"],
+    "allowedCudaVersions": ["13.0"],
     "presets": [
       {
         "name": "deepseek-ai/deepseek-r1-distill-llama-8b",
@@ -621,7 +621,7 @@
           "name": "Enforce Eager",
           "type": "boolean",
           "description": "Always use eager-mode PyTorch. If False (0), will use eager mode and CUDA graph in hybrid for maximal performance and flexibility",
-          "default": false,
+          "default": true,
           "advanced": true
         }
       },
@@ -795,6 +795,26 @@
           "default": "",
           "advanced": true
         }
+      },
+      {
+        "key": "PYTORCH_ALLOC_CONF",
+        "input": {
+          "name": "PyTorch Alloc Config",
+          "type": "string",
+          "description": "PyTorch allocation configuration, remove this if you want to use the default configuration",
+          "default": "expandable_segments:True",
+          "advanced": true
+        }
+      },
+      {
+        "key": "VLLM_USE_DEEP_GEMM",
+        "input": {
+          "name": "Use DeepGEMM",
+          "type": "string",
+          "description": "Enable DeepGEMM FP8 kernels (MoE and MQA logits). Set to 1 to enable, 0 to disable. Required for DeepSeek V4 models. Disabled by default — enable on H100/H200 for potential throughput gains. Some GPUs (e.g. H20) may perform better with this off.",
+          "default": "0",
+          "advanced": true
+        }
       }
     ]
   }

.runpod/tests.json

@@ -5,7 +5,7 @@
       "input": {
         "prompt": "Write a short poem about artificial intelligence."
       },
-      "timeout": 30000
+      "timeout": 300000
     },
     {
       "name": "openai_messages_test",
@@ -26,7 +26,7 @@
           "temperature": 0.1
         }
       },
-      "timeout": 30000
+      "timeout": 300000
     }
   ],
   "config": {
@@ -38,6 +38,6 @@
         "value": "HuggingFaceTB/SmolLM2-135M-Instruct"
       }
     ],
-    "allowedCudaVersions": ["12.9", "12.8", "12.7", "12.6", "12.5"]
+    "allowedCudaVersions": ["13.0"]
   }
 }

Dockerfile

@@ -1,16 +1,17 @@
-FROM nvidia/cuda:12.9.1-base-ubuntu22.04 
+FROM nvidia/cuda:13.0.2-devel-ubuntu22.04
 
 RUN apt-get update -y \
-    && apt-get install -y python3-pip curl \
-    && curl -LsSf https://astral.sh/uv/0.10.9/install.sh  | sh
+    && apt-get install -y python3-pip curl git \
+    && curl -LsSf https://astral.sh/uv/install.sh  | sh
 
 ENV PATH="/root/.local/bin:$PATH"
 
-RUN ldconfig /usr/local/cuda-12.9/compat/
+RUN ldconfig /usr/local/cuda-13.0/compat/
 
-# Install vLLM with FlashInfer - use CUDA 12.9 PyTorch wheels
+# Install vLLM with FlashInfer - use CUDA 130 PyTorch wheels
 RUN uv pip install --system "packaging>=24.2" && \
-    uv pip install --system "vllm[flashinfer]==0.16.0" --extra-index-url https://download.pytorch.org/whl/cu129
+    uv pip install --system "vllm[flashinfer]==0.20.2" && \
+    uv pip install --system git+https://github.com/deepseek-ai/DeepGEMM.git@714dd1a4a980f7937a74343d19a8eba4fe321480 --no-build-isolation
 
 # Install additional Python dependencies (after vLLM to avoid PyTorch version conflicts)
 COPY builder/requirements.txt /requirements.txt
@@ -42,7 +43,9 @@ ENV MODEL_NAME=$MODEL_NAME \
     # Prevent rayon thread pool panic in containers where ulimit -u < nproc
     # (tokenizers uses Rust's rayon which tries to spawn threads = CPU cores)
     TOKENIZERS_PARALLELISM=false \
-    RAYON_NUM_THREADS=4
+    RAYON_NUM_THREADS=4 \
+    # Disable DeepGEMM MoE kernels by default; override with VLLM_USE_DEEP_GEMM=1 to enable
+    VLLM_USE_DEEP_GEMM=0
 
 ENV PYTHONPATH="/:/vllm-workspace"
 

README.md

@@ -8,7 +8,8 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https:
 
 ![vLLM worker banner](https://image.runpod.ai/preview/vllm/vllm-banner.png)
 
-Current vLLM version: [0.16.0](https://github.com/vllm-project/vllm/releases/tag/v0.16.0)
+Current vLLM version: [0.20.2](https://github.com/vllm-project/vllm/releases/tag/v0.20.2)
+
 
 > Check out our Load Balancer implementation here: [vLLM Load Balancer](https://github.com/runpod-workers/vllm-loadbalancer-ep)
 
@@ -46,7 +47,7 @@ Current vLLM version: [0.16.0](https://github.com/vllm-project/vllm/releases/tag
 **📦 Docker Image**: `runpod/worker-v1-vllm:<version>`
 
 - **Available Versions**: See [GitHub Releases](https://github.com/runpod-workers/worker-vllm/releases)
-- **CUDA Compatibility**: Requires CUDA >= 12.1
+- **CUDA Compatibility**: Requires CUDA >= 13.0
 
 ### Configuration
 

builder/requirements.txt

@@ -1,15 +1,15 @@
 ray
 pandas
 pyarrow
-runpod
+runpod==1.9.0
 huggingface-hub
-lmcache==0.4.1
+lmcache==0.4.5
 packaging>=24.2
 typing-extensions>=4.8.0
 pydantic
 pydantic-settings
 hf-transfer
-transformers>=4.57.0,<5
+transformers>=5
 bitsandbytes>=0.45.0
-kernels
+kernels<0.15
 torch-c-dlpack-ext

docs/configuration.md

@@ -97,10 +97,13 @@ If `SPECULATIVE_CONFIG` is set, it takes priority over individual env vars. When
 | `MAX_SEQ_LEN_TO_CAPTURE`       | `8192`  | `int`           | Maximum context length covered by CUDA graphs. When a sequence has context length larger than this, we fall back to eager mode.     |
 | `DISABLE_CUSTOM_ALL_REDUCE`    | `0`     | `int`           | Enables or disables custom all reduce.                                                                                              |
 | `ENABLE_EXPERT_PARALLEL`       | `False` | `bool`          | Enable Expert Parallel for MoE models.                                                                                              |
+| `VLLM_USE_DEEP_GEMM`           | `0`     | `str` (`0`/`1`) | Enable DeepGEMM FP8 kernels for MoE and MQA logits computation. Disabled by default. Must be `"0"` or `"1"` — not `true`/`false`. See note below. |
 | `ATTENTION_BACKEND`            | `None`  | `str`           | Attention backend to use (e.g., `FLASH_ATTN`, `FLASHINFER`, `TRITON_FLASH_ATTN`). Replaces deprecated `VLLM_ATTENTION_BACKEND`.     |
 | `ASYNC_SCHEDULING`             | `None`  | `bool`          | Enable async scheduling (overlaps engine scheduling with GPU execution). Default: enabled in vLLM 0.14.0+. Set to `false` to disable. |
 | `STREAM_INTERVAL`              | `1`     | `int`           | Controls how often to yield streaming results. Lower = more frequent updates.                                                        |
 
+> **Note (`VLLM_USE_DEEP_GEMM`):** DeepGEMM is used in two places: MoE weight computation and MQA logits computation. It is necessary for MQA logits computation on supported hardware — required for DeepSeek V4 models. Set `VLLM_USE_DEEP_GEMM=1` to enable. Set `VLLM_USE_DEEP_GEMM=0` to disable the MoE part and fall back to flashinfer/cutlass FP8 kernels. **Value must be `"0"` or `"1"` — not `"true"`/`"false"`.** Some users report better performance with `VLLM_USE_DEEP_GEMM=0`, particularly on H20 GPUs. Disabling it also skips the DeepGEMM warmup phase, reducing cold-start time. Requires CUDA 13.0+ and SM90+ (H100/H200) to use; the library is installed but inactive by default.
+
 ## Tokenizer Settings
 
 | Variable               | Default | Type/Choices                        | Description                                                                                       |

src/engine.py

@@ -1,4 +1,5 @@
 import asyncio
+import inspect
 import json
 import logging
 import os
@@ -7,6 +8,7 @@ from typing import AsyncGenerator, Optional
 
 from dotenv import load_dotenv
 from vllm import AsyncLLMEngine
+from vllm.inputs import TextPrompt
 from vllm.entrypoints.logger import RequestLogger
 from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest, AnthropicMessagesResponse, AnthropicError, AnthropicErrorResponse
 from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
@@ -19,6 +21,7 @@ from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePat
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
 from vllm.entrypoints.openai.responses.protocol import ResponsesRequest, ResponsesResponse
 from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
+from vllm.entrypoints.serve.render.serving import OpenAIServingRender
 
 from constants import DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE_GROWTH_FACTOR, DEFAULT_MAX_CONCURRENCY, DEFAULT_MIN_BATCH_SIZE
 from engine_args import get_engine_args
@@ -29,20 +32,33 @@ class vLLMEngine:
     def __init__(self, engine = None):
         load_dotenv() # For local development
         self.engine_args = get_engine_args()
-        logging.info(f"Engine args: {self.engine_args}")
-        
-        # Initialize vLLM engine first
-        self.llm = self._initialize_llm() if engine is None else engine.llm
-        
-        # Only create custom tokenizer wrapper if not using mistral tokenizer mode
-        # For mistral models, let vLLM handle tokenizer initialization
-        if self.engine_args.tokenizer_mode != 'mistral':
-            self.tokenizer = TokenizerWrapper(self.engine_args.tokenizer or self.engine_args.model, 
-                                              self.engine_args.tokenizer_revision, 
-                                              self.engine_args.trust_remote_code)
+
+        if engine is None:
+            ea = self.engine_args
+            summary = {
+                "model": ea.model,
+                "dtype": ea.dtype,
+                "quantization": ea.quantization,
+                "max_model_len": ea.max_model_len,
+                "tensor_parallel_size": ea.tensor_parallel_size,
+                "gpu_memory_utilization": ea.gpu_memory_utilization,
+            }
+            if ea.tokenizer and ea.tokenizer != ea.model:
+                summary["tokenizer"] = ea.tokenizer
+            logging.info("Engine config: %s", summary)
+            logging.debug("Full engine args: %s", ea)
+
+            self.llm = self._initialize_llm()
+
+            if self.engine_args.tokenizer_mode != 'mistral':
+                self.tokenizer = TokenizerWrapper(self.engine_args.tokenizer or self.engine_args.model,
+                                                  self.engine_args.tokenizer_revision,
+                                                  self.engine_args.trust_remote_code)
+            else:
+                self.tokenizer = None
         else:
-            # For mistral models, we'll get the tokenizer from vLLM later
-            self.tokenizer = None
+            self.llm = engine.llm
+            self.tokenizer = engine.tokenizer
             
         self.max_concurrency = int(os.getenv("MAX_CONCURRENCY", DEFAULT_MAX_CONCURRENCY))
         self.default_batch_size = int(os.getenv("DEFAULT_BATCH_SIZE", DEFAULT_BATCH_SIZE))
@@ -115,7 +131,7 @@ class vLLMEngine:
         if apply_chat_template or isinstance(llm_input, list):
             tokenizer_wrapper = self._get_tokenizer_for_chat_template()
             llm_input = tokenizer_wrapper.apply_chat_template(llm_input)
-        results_generator = self.llm.generate(llm_input, validated_sampling_params, request_id)
+        results_generator = self.llm.generate(TextPrompt(prompt=llm_input), validated_sampling_params, request_id)
         n_responses, n_input_tokens, is_first_output = validated_sampling_params.n, 0, True
         last_output_texts, token_counters = ["" for _ in range(n_responses)], {"batch": 0, "total": 0}
 
@@ -205,19 +221,48 @@ class OpenAIvLLMEngine(vLLMEngine):
             self.raw_openai_output = bool(int(raw_output_env))
 
     def _load_lora_adapters(self):
-        adapters = []
-        try:
-            adapters = json.loads(os.getenv("LORA_MODULES", '[]'))
-        except Exception as e:
-            logging.info(f"---Initialized adapter json load error: {e}")
+        lora_modules_env = os.getenv("LORA_MODULES", "")
+        if not lora_modules_env:
+            return []
 
-        for i, adapter in enumerate(adapters):
+        try:
+            parsed = json.loads(lora_modules_env)
+        except json.JSONDecodeError as e:
+            logging.error(
+                "LORA_MODULES could not be parsed as JSON: %s — no LoRA adapters loaded. Value: %r",
+                e, lora_modules_env,
+            )
+            return []
+
+        # Accept a single adapter dict as well as an array
+        if isinstance(parsed, dict):
+            parsed = [parsed]
+
+        if not isinstance(parsed, list):
+            logging.error(
+                "LORA_MODULES must be a JSON array of adapter objects, got %s — no LoRA adapters loaded.",
+                type(parsed).__name__,
+            )
+            return []
+
+        adapters = []
+        for i, adapter in enumerate(parsed):
             try:
-                adapters[i] = LoRAModulePath(**adapter)
-                logging.info(f"---Initialized adapter: {adapter}")
+                adapters.append(LoRAModulePath(**adapter))
+                logging.info("Loaded LoRA adapter config [%d]: %s", i, adapter)
             except Exception as e:
-                logging.info(f"---Initialized adapter not worked: {e}")
-                continue
+                logging.error(
+                    "Failed to parse LoRA adapter at index %d: %s. Config: %r",
+                    i, e, adapter,
+                )
+
+        if parsed and not adapters:
+            logging.error(
+                "LORA_MODULES specified %d adapter(s) but none could be loaded — "
+                "OpenAI model name lookups for LoRA adapters will fail.",
+                len(parsed),
+            )
+
         return adapters
 
     async def _ensure_engines_initialized(self):
@@ -246,16 +291,32 @@ class OpenAIvLLMEngine(vLLMEngine):
             lora_modules=self.lora_adapters,
         )
         await self.serving_models.init_static_loras()
-        
+
         # Get chat template from vLLM tokenizer if available
         chat_template = None
         if self.tokenizer and hasattr(self.tokenizer, 'tokenizer'):
             chat_template = self.tokenizer.tokenizer.chat_template
-        
+
+        self.openai_serving_render = OpenAIServingRender(
+            model_config=self.llm.model_config,
+            renderer=self.llm.renderer,
+            model_registry=self.serving_models.registry,
+            request_logger=None,
+            chat_template=chat_template,
+            chat_template_content_format="auto",
+            trust_request_chat_template=os.getenv('TRUST_REQUEST_CHAT_TEMPLATE', 'false').lower() == 'true',
+            enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
+            exclude_tools_when_tool_choice_none=os.getenv('EXCLUDE_TOOLS_WHEN_TOOL_CHOICE_NONE', 'false').lower() == 'true',
+            tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None,
+            reasoning_parser=os.getenv('REASONING_PARSER', "") or None,
+            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
+        )
+
         self.chat_engine = OpenAIServingChat(
-            engine_client=self.llm, 
+            engine_client=self.llm,
             models=self.serving_models,
             response_role=self.response_role,
+            openai_serving_render=self.openai_serving_render,
             request_logger=None,
             chat_template=chat_template,
             chat_template_content_format="auto",
@@ -268,20 +329,20 @@ class OpenAIvLLMEngine(vLLMEngine):
             enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
             enable_log_outputs=os.getenv('ENABLE_LOG_OUTPUTS', 'false').lower() == 'true',
-            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
         )
         self.completion_engine = OpenAIServingCompletion(
             engine_client=self.llm,
             models=self.serving_models,
+            openai_serving_render=self.openai_serving_render,
             request_logger=None,
             return_tokens_as_token_ids=os.getenv('RETURN_TOKENS_AS_TOKEN_IDS', 'false').lower() == 'true',
             enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
-            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
         )
         self.responses_engine = OpenAIServingResponses(
             engine_client=self.llm,
             models=self.serving_models,
+            openai_serving_render=self.openai_serving_render,
             request_logger=None,
             chat_template=chat_template,
             chat_template_content_format="auto",
@@ -293,12 +354,12 @@ class OpenAIvLLMEngine(vLLMEngine):
             enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
             enable_log_outputs=os.getenv('ENABLE_LOG_OUTPUTS', 'false').lower() == 'true',
-            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
         )
         self.messages_engine = AnthropicServingMessages(
             engine_client=self.llm,
             models=self.serving_models,
             response_role=self.response_role,
+            openai_serving_render=self.openai_serving_render,
             request_logger=None,
             chat_template=chat_template,
             chat_template_content_format="auto",
@@ -310,8 +371,11 @@ class OpenAIvLLMEngine(vLLMEngine):
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
         )
 
-        if hasattr(self.chat_engine, 'warmup'):
-            await self.chat_engine.warmup()
+        warmup = getattr(self.chat_engine, 'warmup', None)
+        if callable(warmup):
+            result = warmup()
+            if inspect.isawaitable(result):
+                await result
 
     async def generate(self, openai_request: JobInput):
         # Ensure engines are ready (no-op if already initialized at startup)

src/engine_args.py

@@ -82,6 +82,16 @@ def _convert_env_value_to_field_type(value: str, field_name: str, field_type: ty
         if type(None) in (args or ()):
             return None
         raise ValueError("empty value not allowed for non-optional field")
+
+    # Union[bool, str, ...]: only coerce to bool for unambiguous literals;
+    # otherwise preserve the string (e.g. hf_token="hf_abc..." must stay a str).
+    if get_origin(field_type) is not None:
+        union_types = [a for a in (get_args(field_type) or ()) if a is not type(None)]
+        if bool in union_types and str in union_types:
+            if str(val).lower() in ("true", "false", "1", "0", "yes", "no", "on", "off"):
+                return str(val).lower() in ("true", "1", "yes", "on")
+            return str(val)
+
     effective_type = _resolve_field_type(field_type)
     # bool
     if effective_type is bool:
@@ -342,6 +352,58 @@ def _sanitize_hf_overrides(hf_overrides: dict) -> dict | None:
     return result or None
 
 
+def _resolve_cached_model_path(model_name: str) -> str:
+    """Return a local snapshot path when the HF cache was stored with lowercase names.
+
+    Some model stores (e.g. RunPod pre-cached volumes) normalize repo IDs to
+    lowercase. HuggingFace Hub stores caches as
+    ``models--{org}--{model}/snapshots/{hash}/`` preserving the original casing,
+    so MODEL_NAME=Qwen/Qwen2.5-Coder-32B-Instruct-AWQ will miss a cache stored
+    as ``models--qwen--qwen2.5-coder-32b-instruct-awq/``.
+
+    If the exact-case cache directory is absent but a lowercase variant exists,
+    the latest snapshot path is returned so vLLM loads from disk rather than
+    attempting a redundant download.
+    """
+    if os.path.isabs(model_name):
+        return model_name
+
+    cache_dir = (
+        os.getenv("HUGGINGFACE_HUB_CACHE")
+        or os.getenv("HF_HOME")
+        or os.path.expanduser("~/.cache/huggingface/hub")
+    )
+
+    folder_name = f"models--{model_name.replace('/', '--')}"
+
+    if os.path.isdir(os.path.join(cache_dir, folder_name)):
+        return model_name
+
+    lower_dir = os.path.join(cache_dir, folder_name.lower())
+    if not os.path.isdir(lower_dir):
+        return model_name
+
+    snapshots_dir = os.path.join(lower_dir, "snapshots")
+    if not os.path.isdir(snapshots_dir):
+        return model_name
+
+    try:
+        snapshots = sorted(os.listdir(snapshots_dir))
+    except OSError:
+        return model_name
+
+    if not snapshots:
+        return model_name
+
+    resolved = os.path.join(snapshots_dir, snapshots[-1])
+    logging.info(
+        "MODEL_NAME %r not found at original casing in HF cache; "
+        "resolved to lowercase cached snapshot at %r",
+        model_name, resolved,
+    )
+    return resolved
+
+
 def get_local_args():
     """
     Retrieve local arguments from a JSON file.
@@ -517,4 +579,8 @@ def get_engine_args():
     if speculative_config:
         args["speculative_config"] = speculative_config
 
+    # Resolve lowercase HF cache paths (FDE-174)
+    if args.get("model"):
+        args["model"] = _resolve_cached_model_path(args["model"])
+
     return AsyncEngineArgs(**args)

src/tokenizer.py

@@ -1,10 +1,12 @@
-from transformers import AutoTokenizer
+import logging
 import os
 from typing import Union
 
+from transformers import AutoTokenizer
+
 class TokenizerWrapper:
     def __init__(self, tokenizer_name_or_path, tokenizer_revision, trust_remote_code):
-        print(f"tokenizer_name_or_path: {tokenizer_name_or_path}, tokenizer_revision: {tokenizer_revision}, trust_remote_code: {trust_remote_code}")
+        logging.debug("tokenizer_name_or_path: %s, tokenizer_revision: %s, trust_remote_code: %s", tokenizer_name_or_path, tokenizer_revision, trust_remote_code)
         self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, revision=tokenizer_revision or "main", trust_remote_code=trust_remote_code)
         self.custom_chat_template = os.getenv("CUSTOM_CHAT_TEMPLATE")
         self.has_chat_template = bool(self.tokenizer.chat_template) or bool(self.custom_chat_template)