Merge pull request #293 from runpod-workers/fix/update-deep-gemm-hub-value

fix: update VLLM_USE_DEEP_GEMM hub to default to 0

.github/workflows/CI-runpod_dep.yml

@@ -9,59 +9,60 @@ on:
 
   workflow_dispatch:
 
+permissions:
+  contents: write
+  pull-requests: write
+
 jobs:
   check_dep:
     runs-on: ubuntu-latest
     name: Check python requirements file and update
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Check for new package version and update
         run: |
-          echo "Fetching the current runpod version from requirements.txt..."
+          echo "Fetching current runpod version from requirements.txt..."
 
-          # Get current version, allowing both == and ~= in the search pattern
+          # Match runpod with any version specifier or no specifier at all
-          current_version=$(grep -oP 'runpod[~=]{1,2}\K[^"]+' ./builder/requirements.txt)
+          current_version=$(grep -oP '^runpod([~>=!<]{1,2}\K[\d.]+)?' ./builder/requirements.txt | grep -oP '[\d.]+' || echo "")
-          echo "Current version: $current_version"
+          echo "Current version: ${current_version:-unset}"
 
-          # Extract major and minor from current version
+          echo "Fetching latest runpod version from PyPI..."
-          current_major_minor=$(echo $current_version | cut -d. -f1,2)
+          new_version=$(curl -sf https://pypi.org/pypi/runpod/json | jq -r .info.version)
-          echo "Current major.minor: $current_major_minor"
-
-          echo "Fetching the latest runpod version from PyPI..."
-          
-          # Get new version from PyPI
-          new_version=$(curl -s https://pypi.org/pypi/runpod/json | jq -r .info.version)
           echo "NEW_VERSION_ENV=$new_version" >> $GITHUB_ENV
           echo "New version: $new_version"
 
-          # Extract major and minor from new version
-          new_major_minor=$(echo $new_version | cut -d. -f1,2)
-          echo "New major.minor: $new_major_minor"
-
           if [ -z "$new_version" ]; then
-              echo "ERROR: Failed to fetch the new version from PyPI."
+            echo "ERROR: Failed to fetch new version from PyPI."
-              exit 1
+            exit 1
           fi
 
-          # Check if the major or minor version is different
+          if [ -z "$current_version" ]; then
-          if [ "$current_major_minor" = "$new_major_minor" ]; then
+            echo "No version pin found — pinning to $new_version."
-              echo "No update needed. The new version ($new_major_minor) is within the allowed range (~= $current_major_minor)."
+          else
+            current_major_minor=$(echo "$current_version" | cut -d. -f1,2)
+            new_major_minor=$(echo "$new_version" | cut -d. -f1,2)
+            echo "Current major.minor: $current_major_minor  New major.minor: $new_major_minor"
+
+            if [ "$current_major_minor" = "$new_major_minor" ]; then
+              echo "No update needed. New version ($new_version) is within ~= $current_major_minor range."
               exit 0
+            fi
+
+            echo "New major/minor detected ($new_major_minor). Updating requirements.txt..."
           fi
 
-          echo "New major/minor detected ($new_major_minor). Updating requirements.txt..."
+          # Replace any `runpod`, `runpod==x`, `runpod~=x`, etc. with pinned version
-
+          sed -i "s|^runpod.*|runpod~=$new_version|" ./builder/requirements.txt
-          # Update requirements.txt, preserving the existing constraint type (~= or ==)
+          echo "requirements.txt updated."
-          sed -i "s/runpod[~=][^ ]*/runpod~=$new_version/" ./builder/requirements.txt
-          echo "requirements.txt has been updated."
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v3
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: Update runpod package version
+          commit-message: "chore: update runpod to ${{ env.NEW_VERSION_ENV }}"
-          title: Update runpod package version
+          title: "chore: update runpod to ${{ env.NEW_VERSION_ENV }}"
-          body: The package version has been updated to ${{ env.NEW_VERSION_ENV }}
+          body: The `runpod` package has been updated to `${{ env.NEW_VERSION_ENV }}`.
           branch: runpod-package-update

.github/workflows/release.yml

@@ -3,7 +3,7 @@ name: Release
 on:
   push:
     tags:
-      - "v[0-9]+.[0-9]+.[0-9]+*" # Trigger on version tags like v1.0.0, v2.1.0, etc.
+      - "v[0-9]+.[0-9]+.[0-9]+*"
   workflow_dispatch:
     inputs:
       version:
@@ -53,16 +53,13 @@ jobs:
 
           # Determine version based on trigger type
           if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
-            # Manual trigger: use input version
             VERSION="${{ github.event.inputs.version }}"
-            echo "RELEASE_VERSION=${VERSION}" >> $GITHUB_ENV
+          elif [[ "${{ github.event_name }}" == "release" ]]; then
-            echo "IS_MANUAL_RELEASE=true" >> $GITHUB_ENV
+            VERSION="${{ github.event.release.tag_name }}"
           else
-            # Tag trigger: use tag name (remove refs/tags/ prefix)
             VERSION=${GITHUB_REF#refs/tags/}
-            echo "RELEASE_VERSION=${VERSION}" >> $GITHUB_ENV
-            echo "IS_MANUAL_RELEASE=false" >> $GITHUB_ENV
           fi
+          echo "RELEASE_VERSION=${VERSION}" >> $GITHUB_ENV
 
       - name: Build and push the images to Docker Hub
         uses: docker/bake-action@v2
@@ -76,11 +73,45 @@ jobs:
 
       - name: Release Summary
         run: |
-          echo "🚀 Release completed!"
+          echo "Release completed!"
           echo "Version: ${{ env.RELEASE_VERSION }}"
           echo "Docker Image: ${{ env.DOCKERHUB_REPO }}/${{ env.DOCKERHUB_IMG }}:${{ env.RELEASE_VERSION }}"
-          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+
-            echo "Trigger: Manual workflow dispatch"
+      - name: Fetch Release Notes
-          else
+        run: |
-            echo "Trigger: GitHub release (tag: ${{ github.ref_name }})"
+          RESPONSE=$(curl -sf \
+            -H "Authorization: token ${{ github.token }}" \
+            "https://api.github.com/repos/${{ github.repository }}/releases/tags/${{ env.RELEASE_VERSION }}" 2>/dev/null) || true
+          if [[ -n "$RESPONSE" ]]; then
+            NOTES=$(echo "$RESPONSE" | jq -r '.body // empty')
           fi
+          printf '%s' "${NOTES:-No release notes available.}" > /tmp/release_notes.txt
+
+      - name: Notify Slack
+        run: |
+          jq -n \
+            --arg version "${{ env.RELEASE_VERSION }}" \
+            --arg docker "${{ env.DOCKERHUB_REPO }}/${{ env.DOCKERHUB_IMG }}:${{ env.RELEASE_VERSION }}" \
+            --rawfile notes /tmp/release_notes.txt \
+            --arg url "https://github.com/${{ github.repository }}/releases/tag/${{ env.RELEASE_VERSION }}" \
+            '{
+              text: (":rocket: New :runpod-new-whiteonpurple: Runpod worker-vllm release: *" + $version + "*"),
+              blocks: [
+                {
+                  type: "section",
+                  text: {
+                    type: "mrkdwn",
+                    text: (":banana-dance: *New Release — worker-vllm " + $version + "*\n*Docker:* `" + $docker + "`\n<" + $url + "|View release on GitHub>")
+                  }
+                },
+                {
+                  type: "section",
+                  text: {
+                    type: "mrkdwn",
+                    text: ("*Release Notes:*\n" + $notes)
+                  }
+                }
+              ]
+            }' | curl -sf -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \
+              -H "Content-Type: application/json" \
+              -d @-

.github/workflows/slack-pr-issue-notify.yml

@@ -0,0 +1,41 @@
+name: Slack PR Notifications
+
+on:
+  pull_request:
+    types: [opened]
+  issues:
+    types: [opened]
+
+permissions:
+    contents: read
+   
+jobs:
+  notify:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Slack - New PR
+        if: github.event_name == 'pull_request'
+        run: |
+          curl -sf -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \
+            -H "Content-Type: application/json" \
+            -d '{
+              "text": ":rocket: New PR in worker-vllm: *${{ github.event.pull_request.title }}*",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": ":rocket: *New Pull Request — worker-vllm*\n*<${{ github.event.pull_request.html_url }}|${{ github.event.pull_request.title }}>*\nOpened by *${{ github.event.pull_request.user.login }}*"
+                  }
+                },
+                {
+                  "type": "context",
+                  "elements": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "${{ github.event.pull_request.base.ref }} ← ${{ github.event.pull_request.head.ref }}"
+                    }
+                  ]
+                }
+              ]
+            }'

.github/workflows/slack-vllm-monitor.yml

@@ -0,0 +1,73 @@
+name: Monitor vLLM Releases
+
+on:
+  schedule:
+    - cron: '0 0 * * *' # Every day at midnight
+  workflow_dispatch:
+
+
+permissions:
+    contents: read
+   
+jobs:
+  check-vllm-release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Restore last known vLLM tag
+        uses: actions/cache/restore@v4
+        with:
+          path: .vllm-last-tag
+          key: vllm-tag-${{ github.run_id }}
+          restore-keys: vllm-tag-
+
+      - name: Get latest vLLM release
+        id: vllm
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          response=$(curl -sf https://api.github.com/repos/vllm-project/vllm/releases/latest \
+            -H "Authorization: Bearer $GH_TOKEN")
+          echo "tag=$(echo "$response" | jq -r '.tag_name')" >> $GITHUB_OUTPUT
+          echo "url=$(echo "$response" | jq -r '.html_url')" >> $GITHUB_OUTPUT
+          echo "name=$(echo "$response" | jq -r '.name')" >> $GITHUB_OUTPUT
+
+      - name: Check if new release
+        id: check
+        run: |
+          last=$(cat .vllm-last-tag 2>/dev/null || echo "")
+          current="${{ steps.vllm.outputs.tag }}"
+          echo "Last: $last  Current: $current"
+          if [ -n "$current" ] && [ "$last" != "$current" ]; then
+            echo "is_new=true" >> $GITHUB_OUTPUT
+          else
+            echo "is_new=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Notify Slack
+        if: steps.check.outputs.is_new == 'true'
+        run: |
+          curl -sf -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \
+            -H "Content-Type: application/json" \
+            -d '{
+              "text": ":rocket: New vLLM release: *${{ steps.vllm.outputs.tag }}*",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": ":rocket: *New vLLM Release: ${{ steps.vllm.outputs.tag }}*\n<${{ steps.vllm.outputs.url }}|View on GitHub>"
+                  }
+                }
+              ]
+            }'
+
+      - name: Save new tag
+        if: steps.check.outputs.is_new == 'true'
+        run: echo "${{ steps.vllm.outputs.tag }}" > .vllm-last-tag
+
+      - name: Update cache
+        if: steps.check.outputs.is_new == 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: .vllm-last-tag
+          key: vllm-tag-${{ steps.vllm.outputs.tag }}

.runpod/README.md

@@ -1,4 +1,4 @@
-![vLLM worker banner](https://cpjrphpz3t5wbwfe.public.blob.vercel-storage.com/worker-vllm_banner.jpeg)
+![vLLM worker banner](https://image.runpod.ai/preview/vllm/vllm-banner.png)
 
 Run LLMs using [vLLM](https://docs.vllm.ai) with an OpenAI-compatible API
 
@@ -6,6 +6,8 @@ Run LLMs using [vLLM](https://docs.vllm.ai) with an OpenAI-compatible API
 
 [![RunPod](https://api.runpod.io/badge/runpod-workers/worker-vllm)](https://www.runpod.io/console/hub/runpod-workers/worker-vllm)
 
+Current vLLM version: [0.20.2](https://github.com/vllm-project/vllm/releases/tag/v0.20.2)
+
 ---
 
 ## Endpoint Configuration
@@ -27,11 +29,15 @@ All behaviour is controlled through environment variables:
 | `REASONING_PARSER`                  | Parser for reasoning-capable models               |                     | "deepseek_r1", "qwen3", "granite", "hunyuan_a13b"                  |
 | `OPENAI_SERVED_MODEL_NAME_OVERRIDE` | Override served model name in API                 |                     | String                                                             |
 | `MAX_CONCURRENCY`                   | Maximum concurrent requests                       | 300                 | Integer                                                            |
+| `ENFORCE_EAGER`                     |  If True, we will disable CUDA graph and always execute the model in eager mode. If False, we will use CUDA graph and eager execution in hybrid for maximal performance and flexibility. | true | boolean (true or false) | 
 
 **Pass any vLLM engine arg** not listed above by setting an env var with the **UPPERCASED** field name (e.g. `MAX_MODEL_LEN=4096`, `ENABLE_CHUNKED_PREFILL=true`). The worker auto-discovers all `AsyncEngineArgs` fields from env. See the [vLLM engine args docs](https://docs.vllm.ai/en/latest/configuration/engine_args) for all available options.
 
 For complete configuration options, see the [full configuration documentation](https://github.com/runpod-workers/worker-vllm/blob/main/docs/configuration.md).
 
+### Specify Transformers Version
+To change the version of the [Transformers library](https://github.com/huggingface/transformers) use the `TRANSFORMERS_VERSION` environment variable to specify the version you want to use. Note this might break the handler, so use for development purposes. 
+
 ## API Usage
 
 This worker supports two API formats: **RunPod native** and **OpenAI-compatible**.
@@ -157,6 +163,35 @@ For external clients and SDKs, use the `/openai/v1` path prefix with your RunPod
 {}
 ```
 
+#### OpenAI Responses API
+
+**Path:** `/openai/v1/responses`
+
+Supports the [OpenAI Responses API](https://platform.openai.com/docs/api-reference/responses) format. Note: this route bypasses the RunPod queue and is served directly — use `/openai/` prefixed paths rather than the RunPod job queue for these endpoints.
+
+```json
+{
+  "model": "meta-llama/Llama-3.1-8B-Instruct",
+  "input": "Tell me a joke."
+}
+```
+
+#### Anthropic Messages API
+
+**Path:** `/openai/v1/messages`
+
+Supports the [Anthropic Messages API](https://docs.anthropic.com/en/api/messages) format. Served directly, bypassing the RunPod queue.
+
+```json
+{
+  "model": "meta-llama/Llama-3.1-8B-Instruct",
+  "max_tokens": 256,
+  "messages": [
+    {"role": "user", "content": "Hello!"}
+  ]
+}
+```
+
 #### Response Format
 
 Both APIs return the same response format:
@@ -190,7 +225,7 @@ Minimal Python example using the official `openai` SDK:
 from openai import OpenAI
 import os
 
-# Initialize the OpenAI Client with your RunPod API Key and Endpoint URL
+# Initialize the OpenAI Client with your Runpod API Key and Endpoint URL
 client = OpenAI(
     api_key=os.getenv("RUNPOD_API_KEY"),
     base_url=f"https://api.runpod.ai/v2/<ENDPOINT_ID>/openai/v1",

.runpod/hub.json

@@ -9,7 +9,7 @@
     "containerDiskInGb": 150,
     "gpuIds": "ADA_80_PRO,AMPERE_80",
     "gpuCount": 1,
-    "allowedCudaVersions": ["12.9", "12.8"],
+    "allowedCudaVersions": ["13.0"],
     "presets": [
       {
         "name": "deepseek-ai/deepseek-r1-distill-llama-8b",
@@ -621,7 +621,7 @@
           "name": "Enforce Eager",
           "type": "boolean",
           "description": "Always use eager-mode PyTorch. If False (0), will use eager mode and CUDA graph in hybrid for maximal performance and flexibility",
-          "default": false,
+          "default": true,
           "advanced": true
         }
       },
@@ -795,6 +795,26 @@
           "default": "",
           "advanced": true
         }
+      },
+      {
+        "key": "PYTORCH_ALLOC_CONF",
+        "input": {
+          "name": "PyTorch Alloc Config",
+          "type": "string",
+          "description": "PyTorch allocation configuration, remove this if you want to use the default configuration",
+          "default": "expandable_segments:True",
+          "advanced": true
+        }
+      },
+      {
+        "key": "VLLM_USE_DEEP_GEMM",
+        "input": {
+          "name": "Use DeepGEMM",
+          "type": "string",
+          "description": "Enable DeepGEMM FP8 kernels (MoE and MQA logits). Set to 1 to enable, 0 to disable. Required for DeepSeek V4 models. Disabled by default — enable on H100/H200 for potential throughput gains. Some GPUs (e.g. H20) may perform better with this off.",
+          "default": "0",
+          "advanced": true
+        }
       }
     ]
   }

Dockerfile

@@ -1,20 +1,22 @@
-FROM nvidia/cuda:12.9.1-base-ubuntu22.04 
+FROM nvidia/cuda:13.0.2-devel-ubuntu22.04
 
 RUN apt-get update -y \
-    && apt-get install -y python3-pip
+    && apt-get install -y python3-pip curl git \
+    && curl -LsSf https://astral.sh/uv/install.sh  | sh
 
-RUN ldconfig /usr/local/cuda-12.9/compat/
+ENV PATH="/root/.local/bin:$PATH"
-
-# Install vLLM with FlashInfer from the CUDA 12.9 wheel index.
-RUN python3 -m pip install --upgrade pip && \
-    python3 -m pip install "vllm[flashinfer]==0.17.0" --extra-index-url https://download.pytorch.org/whl/cu129
 
+RUN ldconfig /usr/local/cuda-13.0/compat/
 
+# Install vLLM with FlashInfer - use CUDA 130 PyTorch wheels
+RUN uv pip install --system "packaging>=24.2" && \
+    uv pip install --system "vllm[flashinfer]==0.20.2" && \
+    uv pip install --system git+https://github.com/deepseek-ai/DeepGEMM.git@714dd1a4a980f7937a74343d19a8eba4fe321480 --no-build-isolation
 
 # Install additional Python dependencies (after vLLM to avoid PyTorch version conflicts)
 COPY builder/requirements.txt /requirements.txt
-RUN --mount=type=cache,target=/root/.cache/pip \
+RUN --mount=type=cache,target=/root/.cache/uv \
-    python3 -m pip install --upgrade -r /requirements.txt
+    uv pip install --system -r /requirements.txt
 
 # Setup for Option 2: Building the Image with the Model included
 ARG MODEL_NAME=""
@@ -41,17 +43,20 @@ ENV MODEL_NAME=$MODEL_NAME \
     # Prevent rayon thread pool panic in containers where ulimit -u < nproc
     # (tokenizers uses Rust's rayon which tries to spawn threads = CPU cores)
     TOKENIZERS_PARALLELISM=false \
-    RAYON_NUM_THREADS=4
+    RAYON_NUM_THREADS=4 \
+    # Disable DeepGEMM MoE kernels by default; override with VLLM_USE_DEEP_GEMM=1 to enable
+    VLLM_USE_DEEP_GEMM=0
 
 ENV PYTHONPATH="/:/vllm-workspace"
 
 RUN if [ "${VLLM_NIGHTLY}" = "true" ]; then \
-    pip install -U vllm --pre --index-url https://pypi.org/simple --extra-index-url https://wheels.vllm.ai/nightly && \
+    uv pip install --system -U vllm --pre --index-url https://pypi.org/simple --extra-index-url https://wheels.vllm.ai/nightly && \
     apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* && \
-    pip install git+https://github.com/huggingface/transformers.git; \
+    uv pip install --system git+https://github.com/huggingface/transformers.git; \
 fi
 
 COPY src /src
+RUN chmod +x /src/start.sh
 RUN --mount=type=secret,id=HF_TOKEN,required=false \
     if [ -f /run/secrets/HF_TOKEN ]; then \
     export HF_TOKEN=$(cat /run/secrets/HF_TOKEN); \
@@ -61,4 +66,4 @@ RUN --mount=type=secret,id=HF_TOKEN,required=false \
     fi
 
 # Start the handler
-CMD ["python3", "/src/handler.py"]
+CMD ["/bin/bash", "/src/start.sh"]

README.md

@@ -2,10 +2,17 @@
 
 # OpenAI-Compatible vLLM Serverless Endpoint Worker
 
-Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https://github.com/vllm-project/vllm) Inference Engine on RunPod Serverless with just a few clicks.
+Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https://github.com/vllm-project/vllm) Inference Engine on Runpod Serverless with just a few clicks.
 
 </div>
 
+![vLLM worker banner](https://image.runpod.ai/preview/vllm/vllm-banner.png)
+
+Current vLLM version: [0.20.2](https://github.com/vllm-project/vllm/releases/tag/v0.20.2)
+
+
+> Check out our Load Balancer implementation here: [vLLM Load Balancer](https://github.com/runpod-workers/vllm-loadbalancer-ep)
+
 ## Table of Contents
 
 - [Setting up the Serverless Worker](#setting-up-the-serverless-worker)
@@ -21,9 +28,11 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https:
   - [Modifying your OpenAI Codebase to use your deployed vLLM Worker](#modifying-your-openai-codebase-to-use-your-deployed-vllm-worker)
   - [OpenAI Request Input Parameters](#openai-request-input-parameters)
   - [Chat Completions [RECOMMENDED]](#chat-completions-recommended)
-  - [Examples: Using your RunPod endpoint with OpenAI](#examples-using-your-runpod-endpoint-with-openai)
+  - [Examples: Using your Runpod endpoint with OpenAI](#examples-using-your-runpod-endpoint-with-openai)
     - [Chat Completions](#chat-completions)
     - [Getting a list of names for available models](#getting-a-list-of-names-for-available-models)
+    - [OpenAI Responses API](#openai-responses-api)
+    - [Anthropic Messages API](#anthropic-messages-api)
 - [Usage: Standard (Non-OpenAI)](#usage-standard-non-openai)
   - [Request Input Parameters](#request-input-parameters)
   - [Sampling Parameters](#sampling-parameters)
@@ -33,12 +42,12 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https:
 
 ## Option 1: Deploy Any Model Using Pre-Built Docker Image [Recommended]
 
-**🚀 Deploy Guide**: Follow our [step-by-step deployment guide](https://docs.runpod.io/serverless/vllm/get-started) to deploy using the RunPod Console.
+**🚀 Deploy Guide**: Follow our [step-by-step deployment guide](https://docs.runpod.io/serverless/vllm/get-started) to deploy using the Runpod Console.
 
 **📦 Docker Image**: `runpod/worker-v1-vllm:<version>`
 
 - **Available Versions**: See [GitHub Releases](https://github.com/runpod-workers/worker-vllm/releases)
-- **CUDA Compatibility**: Requires CUDA >= 12.1
+- **CUDA Compatibility**: Requires CUDA >= 13.0
 
 ### Configuration
 
@@ -71,6 +80,10 @@ Any env var whose name matches a valid `AsyncEngineArgs` field (uppercased) is a
 
 For the complete list of all available environment variables, examples, and detailed descriptions: **[Configuration](docs/configuration.md)**
 
+### Specify Transformers Version
+To change the version of the [Transformers library](https://github.com/huggingface/transformers) use the `TRANSFORMERS_VERSION` environment variable to specify the version you want to use. Note this might break the handler, so use for development purposes. 
+
+
 ## Option 2: Build Docker Image with Model Inside
 
 To build an image with the model baked in, you must specify the following docker arguments when building the image.
@@ -142,13 +155,13 @@ You can deploy **any model on Hugging Face** that is supported by vLLM. For the
 
 # Usage: OpenAI Compatibility
 
-The vLLM Worker is fully compatible with OpenAI's API, and you can use it with any OpenAI Codebase by changing only 3 lines in total. The supported routes are <ins>Chat Completions</ins> and <ins>Models</ins> - with both streaming and non-streaming.
+The vLLM Worker is fully compatible with OpenAI's API, and you can use it with any OpenAI Codebase by changing only 3 lines in total. The supported routes are <ins>Chat Completions</ins>, <ins>Models</ins>, <ins>Responses</ins>, and <ins>Messages</ins> - with both streaming and non-streaming.
 
 ## Modifying your OpenAI Codebase to use your deployed vLLM Worker
 
 **Python** (similar to Node.js, etc.):
 
-1. When initializing the OpenAI Client in your code, change the `api_key` to your RunPod API Key and the `base_url` to your RunPod Serverless Endpoint URL in the following format: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1`, filling in your deployed endpoint ID. For example, if your Endpoint ID is `abc1234`, the URL would be `https://api.runpod.ai/v2/abc1234/openai/v1`.
+1. When initializing the OpenAI Client in your code, change the `api_key` to your Runpod API Key and the `base_url` to your Runpod Serverless Endpoint URL in the following format: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1`, filling in your deployed endpoint ID. For example, if your Endpoint ID is `abc1234`, the URL would be `https://api.runpod.ai/v2/abc1234/openai/v1`.
 
    - Before:
 
@@ -174,7 +187,7 @@ The vLLM Worker is fully compatible with OpenAI's API, and you can use it with a
    ```python
    response = client.chat.completions.create(
        model="gpt-3.5-turbo",
-       messages=[{"role": "user", "content": "Why is RunPod the best platform?"}],
+       messages=[{"role": "user", "content": "Why is Runpod the best platform?"}],
        temperature=0,
        max_tokens=100,
    )
@@ -183,7 +196,7 @@ The vLLM Worker is fully compatible with OpenAI's API, and you can use it with a
    ```python
    response = client.chat.completions.create(
        model="<YOUR DEPLOYED MODEL REPO/NAME>",
-       messages=[{"role": "user", "content": "Why is RunPod the best platform?"}],
+       messages=[{"role": "user", "content": "Why is Runpod the best platform?"}],
        temperature=0,
        max_tokens=100,
    )
@@ -191,7 +204,7 @@ The vLLM Worker is fully compatible with OpenAI's API, and you can use it with a
 
 **Using http requests**:
 
-1. Change the `Authorization` header to your RunPod API Key and the `url` to your RunPod Serverless Endpoint URL in the following format: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1`
+1. Change the `Authorization` header to your Runpod API Key and the `url` to your Runpod Serverless Endpoint URL in the following format: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1`
    - Before:
    ```bash
    curl https://api.openai.com/v1/chat/completions \
@@ -202,7 +215,7 @@ The vLLM Worker is fully compatible with OpenAI's API, and you can use it with a
    "messages": [
      {
        "role": "user",
-       "content": "Why is RunPod the best platform?"
+       "content": "Why is Runpod the best platform?"
      }
    ],
    "temperature": 0,
@@ -219,7 +232,7 @@ The vLLM Worker is fully compatible with OpenAI's API, and you can use it with a
    "messages": [
      {
        "role": "user",
-       "content": "Why is RunPod the best platform?"
+       "content": "Why is Runpod the best platform?"
      }
    ],
    "temperature": 0,
@@ -239,7 +252,7 @@ When using the chat completion feature of the vLLM Serverless Endpoint Worker, y
 | Parameter           | Type                             | Default Value | Description                                                                                                                                                                                                                                                  |
 | ------------------- | -------------------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 | `messages`          | Union[str, List[Dict[str, str]]] |               | List of messages, where each message is a dictionary with a `role` and `content`. The model's chat template will be applied to the messages automatically, so the model must have one or it should be specified as `CUSTOM_CHAT_TEMPLATE` env var.           |
-| `model`             | str                              |               | The model repo that you've deployed on your RunPod Serverless Endpoint. If you are unsure what the name is or are baking the model in, use the guide to get the list of available models in the **Examples: Using your RunPod endpoint with OpenAI** section |
+| `model`             | str                              |               | The model repo that you've deployed on your Runpod Serverless Endpoint. If you are unsure what the name is or are baking the model in, use the guide to get the list of available models in the **Examples: Using your Runpod endpoint with OpenAI** section |
 | `temperature`       | Optional[float]                  | 0.7           | Float that controls the randomness of the sampling. Lower values make the model more deterministic, while higher values make the model more random. Zero means greedy sampling.                                                                              |
 | `top_p`             | Optional[float]                  | 1.0           | Float that controls the cumulative probability of the top tokens to consider. Must be in (0, 1]. Set to 1 to consider all tokens.                                                                                                                            |
 | `n`                 | Optional[int]                    | 1             | Number of output sequences to return for the given prompt.                                                                                                                                                                                                   |
@@ -269,15 +282,15 @@ Additional parameters supported by vLLM:
 
 </details>
 
-### Examples: Using your RunPod endpoint with OpenAI
+### Examples: Using your Runpod endpoint with OpenAI
 
-First, initialize the OpenAI Client with your RunPod API Key and Endpoint URL:
+First, initialize the OpenAI Client with your Runpod API Key and Endpoint URL:
 
 ```python
 from openai import OpenAI
 import os
 
-# Initialize the OpenAI Client with your RunPod API Key and Endpoint URL
+# Initialize the OpenAI Client with your Runpod API Key and Endpoint URL
 client = OpenAI(
     api_key=os.environ.get("RUNPOD_API_KEY"),
     base_url="https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1",
@@ -293,7 +306,7 @@ This is the format used for GPT-4 and focused on instruction-following and chat.
   # Create a chat completion stream
   response_stream = client.chat.completions.create(
       model="<YOUR DEPLOYED MODEL REPO/NAME>",
-      messages=[{"role": "user", "content": "Why is RunPod the best platform?"}],
+      messages=[{"role": "user", "content": "Why is Runpod the best platform?"}],
       temperature=0,
       max_tokens=100,
       stream=True,
@@ -307,7 +320,7 @@ This is the format used for GPT-4 and focused on instruction-following and chat.
   # Create a chat completion
   response = client.chat.completions.create(
       model="<YOUR DEPLOYED MODEL REPO/NAME>",
-      messages=[{"role": "user", "content": "Why is RunPod the best platform?"}],
+      messages=[{"role": "user", "content": "Why is Runpod the best platform?"}],
       temperature=0,
       max_tokens=100,
   )
@@ -325,6 +338,62 @@ list_of_models = [model.id for model in models_response]
 print(list_of_models)
 ```
 
+### OpenAI Responses API
+
+**Path:** `/openai/v1/responses` (full URL: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1/responses`)
+
+Supports the [OpenAI Responses API](https://platform.openai.com/docs/api-reference/responses) request shape. Like other `/openai/` routes, this is served directly—use the `/openai/` prefix rather than the RunPod native job queue for these calls.
+
+```json
+{
+  "model": "meta-llama/Llama-3.1-8B-Instruct",
+  "input": "Tell me a joke."
+}
+```
+
+**Using HTTP requests:**
+
+```bash
+curl https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1/responses \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR RUNPOD API KEY>" \
+  -d '{
+    "model": "<YOUR DEPLOYED MODEL REPO/NAME>",
+    "input": "Tell me a joke."
+  }'
+```
+
+### Anthropic Messages API
+
+**Path:** `/openai/v1/messages` (full URL: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1/messages`)
+
+Supports the [Anthropic Messages API](https://docs.anthropic.com/en/api/messages) format. Served directly, bypassing the RunPod queue.
+
+```json
+{
+  "model": "meta-llama/Llama-3.1-8B-Instruct",
+  "max_tokens": 256,
+  "messages": [
+    {"role": "user", "content": "Hello!"}
+  ]
+}
+```
+
+**Using HTTP requests:**
+
+```bash
+curl https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1/messages \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR RUNPOD API KEY>" \
+  -d '{
+    "model": "<YOUR DEPLOYED MODEL REPO/NAME>",
+    "max_tokens": 256,
+    "messages": [
+      {"role": "user", "content": "Hello!"}
+    ]
+  }'
+```
+
 # Usage: Standard (Non-OpenAI)
 
 ## Request Input Parameters

builder/requirements.txt

@@ -1,14 +1,15 @@
 ray
 pandas
 pyarrow
-runpod
+runpod==1.9.0
 huggingface-hub
-packaging
+lmcache==0.4.5
+packaging>=24.2
 typing-extensions>=4.8.0
 pydantic
 pydantic-settings
 hf-transfer
-transformers>=4.57.0
+transformers>=5
 bitsandbytes>=0.45.0
 kernels
 torch-c-dlpack-ext

docs/configuration.md

@@ -97,10 +97,13 @@ If `SPECULATIVE_CONFIG` is set, it takes priority over individual env vars. When
 | `MAX_SEQ_LEN_TO_CAPTURE`       | `8192`  | `int`           | Maximum context length covered by CUDA graphs. When a sequence has context length larger than this, we fall back to eager mode.     |
 | `DISABLE_CUSTOM_ALL_REDUCE`    | `0`     | `int`           | Enables or disables custom all reduce.                                                                                              |
 | `ENABLE_EXPERT_PARALLEL`       | `False` | `bool`          | Enable Expert Parallel for MoE models.                                                                                              |
+| `VLLM_USE_DEEP_GEMM`           | `0`     | `str` (`0`/`1`) | Enable DeepGEMM FP8 kernels for MoE and MQA logits computation. Disabled by default. Must be `"0"` or `"1"` — not `true`/`false`. See note below. |
 | `ATTENTION_BACKEND`            | `None`  | `str`           | Attention backend to use (e.g., `FLASH_ATTN`, `FLASHINFER`, `TRITON_FLASH_ATTN`). Replaces deprecated `VLLM_ATTENTION_BACKEND`.     |
 | `ASYNC_SCHEDULING`             | `None`  | `bool`          | Enable async scheduling (overlaps engine scheduling with GPU execution). Default: enabled in vLLM 0.14.0+. Set to `false` to disable. |
 | `STREAM_INTERVAL`              | `1`     | `int`           | Controls how often to yield streaming results. Lower = more frequent updates.                                                        |
 
+> **Note (`VLLM_USE_DEEP_GEMM`):** DeepGEMM is used in two places: MoE weight computation and MQA logits computation. It is necessary for MQA logits computation on supported hardware — required for DeepSeek V4 models. Set `VLLM_USE_DEEP_GEMM=1` to enable. Set `VLLM_USE_DEEP_GEMM=0` to disable the MoE part and fall back to flashinfer/cutlass FP8 kernels. **Value must be `"0"` or `"1"` — not `"true"`/`"false"`.** Some users report better performance with `VLLM_USE_DEEP_GEMM=0`, particularly on H20 GPUs. Disabling it also skips the DeepGEMM warmup phase, reducing cold-start time. Requires CUDA 13.0+ and SM90+ (H100/H200) to use; the library is installed but inactive by default.
+
 ## Tokenizer Settings
 
 | Variable               | Default | Type/Choices                        | Description                                                                                       |

src/engine.py

@@ -1,4 +1,5 @@
 import asyncio
+import inspect
 import json
 import logging
 import os
@@ -7,7 +8,10 @@ from typing import AsyncGenerator, Optional
 
 from dotenv import load_dotenv
 from vllm import AsyncLLMEngine
+from vllm.inputs import TextPrompt
 from vllm.entrypoints.logger import RequestLogger
+from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest, AnthropicMessagesResponse, AnthropicError, AnthropicErrorResponse
+from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
 from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
 from vllm.entrypoints.openai.completion.protocol import CompletionRequest
@@ -15,6 +19,9 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
 from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest, ResponsesResponse
+from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
+from vllm.entrypoints.serve.render.serving import OpenAIServingRender
 
 from constants import DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE_GROWTH_FACTOR, DEFAULT_MAX_CONCURRENCY, DEFAULT_MIN_BATCH_SIZE
 from engine_args import get_engine_args
@@ -25,20 +32,33 @@ class vLLMEngine:
     def __init__(self, engine = None):
         load_dotenv() # For local development
         self.engine_args = get_engine_args()
-        logging.info(f"Engine args: {self.engine_args}")
 
-        # Initialize vLLM engine first
+        if engine is None:
-        self.llm = self._initialize_llm() if engine is None else engine.llm
+            ea = self.engine_args
+            summary = {
+                "model": ea.model,
+                "dtype": ea.dtype,
+                "quantization": ea.quantization,
+                "max_model_len": ea.max_model_len,
+                "tensor_parallel_size": ea.tensor_parallel_size,
+                "gpu_memory_utilization": ea.gpu_memory_utilization,
+            }
+            if ea.tokenizer and ea.tokenizer != ea.model:
+                summary["tokenizer"] = ea.tokenizer
+            logging.info("Engine config: %s", summary)
+            logging.debug("Full engine args: %s", ea)
 
-        # Only create custom tokenizer wrapper if not using mistral tokenizer mode
+            self.llm = self._initialize_llm()
-        # For mistral models, let vLLM handle tokenizer initialization
+
-        if self.engine_args.tokenizer_mode != 'mistral':
+            if self.engine_args.tokenizer_mode != 'mistral':
-            self.tokenizer = TokenizerWrapper(self.engine_args.tokenizer or self.engine_args.model, 
+                self.tokenizer = TokenizerWrapper(self.engine_args.tokenizer or self.engine_args.model,
-                                              self.engine_args.tokenizer_revision, 
+                                                  self.engine_args.tokenizer_revision,
-                                              self.engine_args.trust_remote_code)
+                                                  self.engine_args.trust_remote_code)
+            else:
+                self.tokenizer = None
         else:
-            # For mistral models, we'll get the tokenizer from vLLM later
+            self.llm = engine.llm
-            self.tokenizer = None
+            self.tokenizer = engine.tokenizer
             
         self.max_concurrency = int(os.getenv("MAX_CONCURRENCY", DEFAULT_MAX_CONCURRENCY))
         self.default_batch_size = int(os.getenv("DEFAULT_BATCH_SIZE", DEFAULT_BATCH_SIZE))
@@ -111,7 +131,7 @@ class vLLMEngine:
         if apply_chat_template or isinstance(llm_input, list):
             tokenizer_wrapper = self._get_tokenizer_for_chat_template()
             llm_input = tokenizer_wrapper.apply_chat_template(llm_input)
-        results_generator = self.llm.generate(llm_input, validated_sampling_params, request_id)
+        results_generator = self.llm.generate(TextPrompt(prompt=llm_input), validated_sampling_params, request_id)
         n_responses, n_input_tokens, is_first_output = validated_sampling_params.n, 0, True
         last_output_texts, token_counters = ["" for _ in range(n_responses)], {"batch": 0, "total": 0}
 
@@ -201,19 +221,48 @@ class OpenAIvLLMEngine(vLLMEngine):
             self.raw_openai_output = bool(int(raw_output_env))
 
     def _load_lora_adapters(self):
-        adapters = []
+        lora_modules_env = os.getenv("LORA_MODULES", "")
-        try:
+        if not lora_modules_env:
-            adapters = json.loads(os.getenv("LORA_MODULES", '[]'))
+            return []
-        except Exception as e:
-            logging.info(f"---Initialized adapter json load error: {e}")
 
-        for i, adapter in enumerate(adapters):
+        try:
+            parsed = json.loads(lora_modules_env)
+        except json.JSONDecodeError as e:
+            logging.error(
+                "LORA_MODULES could not be parsed as JSON: %s — no LoRA adapters loaded. Value: %r",
+                e, lora_modules_env,
+            )
+            return []
+
+        # Accept a single adapter dict as well as an array
+        if isinstance(parsed, dict):
+            parsed = [parsed]
+
+        if not isinstance(parsed, list):
+            logging.error(
+                "LORA_MODULES must be a JSON array of adapter objects, got %s — no LoRA adapters loaded.",
+                type(parsed).__name__,
+            )
+            return []
+
+        adapters = []
+        for i, adapter in enumerate(parsed):
             try:
-                adapters[i] = LoRAModulePath(**adapter)
+                adapters.append(LoRAModulePath(**adapter))
-                logging.info(f"---Initialized adapter: {adapter}")
+                logging.info("Loaded LoRA adapter config [%d]: %s", i, adapter)
             except Exception as e:
-                logging.info(f"---Initialized adapter not worked: {e}")
+                logging.error(
-                continue
+                    "Failed to parse LoRA adapter at index %d: %s. Config: %r",
+                    i, e, adapter,
+                )
+
+        if parsed and not adapters:
+            logging.error(
+                "LORA_MODULES specified %d adapter(s) but none could be loaded — "
+                "OpenAI model name lookups for LoRA adapters will fail.",
+                len(parsed),
+            )
+
         return adapters
 
     async def _ensure_engines_initialized(self):
@@ -248,10 +297,26 @@ class OpenAIvLLMEngine(vLLMEngine):
         if self.tokenizer and hasattr(self.tokenizer, 'tokenizer'):
             chat_template = self.tokenizer.tokenizer.chat_template
 
+        self.openai_serving_render = OpenAIServingRender(
+            model_config=self.llm.model_config,
+            renderer=self.llm.renderer,
+            model_registry=self.serving_models.registry,
+            request_logger=None,
+            chat_template=chat_template,
+            chat_template_content_format="auto",
+            trust_request_chat_template=os.getenv('TRUST_REQUEST_CHAT_TEMPLATE', 'false').lower() == 'true',
+            enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
+            exclude_tools_when_tool_choice_none=os.getenv('EXCLUDE_TOOLS_WHEN_TOOL_CHOICE_NONE', 'false').lower() == 'true',
+            tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None,
+            reasoning_parser=os.getenv('REASONING_PARSER', "") or None,
+            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
+        )
+
         self.chat_engine = OpenAIServingChat(
             engine_client=self.llm,
             models=self.serving_models,
             response_role=self.response_role,
+            openai_serving_render=self.openai_serving_render,
             request_logger=None,
             chat_template=chat_template,
             chat_template_content_format="auto",
@@ -264,20 +329,53 @@ class OpenAIvLLMEngine(vLLMEngine):
             enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
             enable_log_outputs=os.getenv('ENABLE_LOG_OUTPUTS', 'false').lower() == 'true',
-            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
         )
         self.completion_engine = OpenAIServingCompletion(
             engine_client=self.llm,
             models=self.serving_models,
+            openai_serving_render=self.openai_serving_render,
             request_logger=None,
             return_tokens_as_token_ids=os.getenv('RETURN_TOKENS_AS_TOKEN_IDS', 'false').lower() == 'true',
             enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
-            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
+        )
+        self.responses_engine = OpenAIServingResponses(
+            engine_client=self.llm,
+            models=self.serving_models,
+            openai_serving_render=self.openai_serving_render,
+            request_logger=None,
+            chat_template=chat_template,
+            chat_template_content_format="auto",
+            return_tokens_as_token_ids=os.getenv('RETURN_TOKENS_AS_TOKEN_IDS', 'false').lower() == 'true',
+            reasoning_parser=os.getenv('REASONING_PARSER', "") or "",
+            enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
+            tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None,
+            tool_server=None,
+            enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
+            enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
+            enable_log_outputs=os.getenv('ENABLE_LOG_OUTPUTS', 'false').lower() == 'true',
+        )
+        self.messages_engine = AnthropicServingMessages(
+            engine_client=self.llm,
+            models=self.serving_models,
+            response_role=self.response_role,
+            openai_serving_render=self.openai_serving_render,
+            request_logger=None,
+            chat_template=chat_template,
+            chat_template_content_format="auto",
+            return_tokens_as_token_ids=os.getenv('RETURN_TOKENS_AS_TOKEN_IDS', 'false').lower() == 'true',
+            reasoning_parser=os.getenv('REASONING_PARSER', "") or "",
+            enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
+            tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None,
+            enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
+            enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
         )
 
-        if hasattr(self.chat_engine, 'warmup'):
+        warmup = getattr(self.chat_engine, 'warmup', None)
-            await self.chat_engine.warmup()
+        if callable(warmup):
+            result = warmup()
+            if inspect.isawaitable(result):
+                await result
 
     async def generate(self, openai_request: JobInput):
         # Ensure engines are ready (no-op if already initialized at startup)
@@ -288,6 +386,12 @@ class OpenAIvLLMEngine(vLLMEngine):
         elif openai_request.openai_route in ["/v1/chat/completions", "/v1/completions"]:
             async for response in self._handle_chat_or_completion_request(openai_request):
                 yield response
+        elif openai_request.openai_route == "/v1/responses":
+            async for response in self._handle_responses_request(openai_request):
+                yield response
+        elif openai_request.openai_route == "/v1/messages":
+            async for response in self._handle_messages_request(openai_request):
+                yield response
         else:
             yield create_error_response("Invalid route").model_dump()
     
@@ -343,3 +447,126 @@ class OpenAIvLLMEngine(vLLMEngine):
                     batch = "".join(batch)
                 yield batch
 
+    async def _handle_responses_request(self, openai_request: JobInput):
+        request_id = getattr(openai_request, "request_id", "unknown")
+
+        try:
+            request = ResponsesRequest(**openai_request.openai_input)
+        except Exception as e:
+            logging.error(
+                "Invalid ResponsesRequest JSON: %s",
+                e,
+                extra={"request_id": request_id}
+            )
+            yield create_error_response(
+                "Invalid request format",
+                err_type="BadRequestError"
+            ).model_dump()
+            return
+
+        dummy_request = DummyRequest()
+        try:
+            response = await self.responses_engine.create_responses(request, raw_request=dummy_request)
+        except Exception as e:
+            logging.error(
+                "Failed to create Responses: %s",
+                e,
+                extra={"request_id": request_id},
+                exc_info=True
+            )
+            yield create_error_response(
+                "Internal server error during response generation",
+                err_type="InternalServerError"
+            ).model_dump()
+            return
+
+        if isinstance(response, (ErrorResponse, ResponsesResponse)):
+            yield response.model_dump()
+            return
+
+        try:
+            async for event in response:
+                if not hasattr(event, "type"):
+                    continue
+                event_type = getattr(event, "type", "unknown")
+                yield f"event: {event_type}\ndata: {event.model_dump_json(indent=None)}\n\n"
+        except Exception as e:
+            logging.error(
+                "Error processing responses stream: %s",
+                e,
+                extra={"request_id": request_id},
+                exc_info=True
+            )
+            error_payload = create_error_response(
+                "Streaming response failed",
+                err_type="InternalServerError"
+            ).model_dump_json()
+            yield f"event: error\ndata: {error_payload}\n\n"
+
+    async def _handle_messages_request(self, openai_request: JobInput):
+        request_id = getattr(openai_request, "request_id", "unknown")
+
+        try:
+            request = AnthropicMessagesRequest(**openai_request.openai_input)
+        except Exception as e:
+            logging.error(
+                "Invalid AnthropicMessagesRequest: %s",
+                e,
+                extra={"request_id": request_id}
+            )
+            yield AnthropicErrorResponse(
+                error=AnthropicError(
+                    type="invalid_request_error",
+                    message="Invalid request format"
+                )
+            ).model_dump()
+            return
+
+        dummy_request = DummyRequest()
+
+        try:
+            response = await self.messages_engine.create_messages(request, raw_request=dummy_request)
+        except Exception as e:
+            logging.error(
+                "Failed to create messages: %s",
+                e,
+                extra={"request_id": request_id},
+                exc_info=True
+            )
+            yield AnthropicErrorResponse(
+                error=AnthropicError(
+                    type="internal_error",
+                    message="Failed to generate messages"
+                )
+            ).model_dump()
+            return
+
+        if isinstance(response, ErrorResponse):
+            error_type = getattr(response, "type", "internal_error")
+            error_message = getattr(response, "message", "Unknown error")
+            yield AnthropicErrorResponse(
+                error=AnthropicError(type=error_type, message=error_message)
+            ).model_dump()
+            return
+
+        if isinstance(response, AnthropicMessagesResponse):
+            yield response.model_dump(exclude_none=True)
+            return
+
+        try:
+            async for chunk in response:
+                yield chunk
+        except Exception as e:
+            logging.error(
+                "Error streaming messages: %s",
+                e,
+                extra={"request_id": request_id},
+                exc_info=True
+            )
+            error_payload = AnthropicErrorResponse(
+                error=AnthropicError(
+                    type="internal_error",
+                    message="Error while streaming messages"
+                )
+            ).model_dump_json()
+            yield f"event: error\ndata: {error_payload}\n\n"

src/engine_args.py

@@ -1,3 +1,4 @@
+import ast
 import os
 import json
 import logging
@@ -81,6 +82,16 @@ def _convert_env_value_to_field_type(value: str, field_name: str, field_type: ty
         if type(None) in (args or ()):
             return None
         raise ValueError("empty value not allowed for non-optional field")
+
+    # Union[bool, str, ...]: only coerce to bool for unambiguous literals;
+    # otherwise preserve the string (e.g. hf_token="hf_abc..." must stay a str).
+    if get_origin(field_type) is not None:
+        union_types = [a for a in (get_args(field_type) or ()) if a is not type(None)]
+        if bool in union_types and str in union_types:
+            if str(val).lower() in ("true", "false", "1", "0", "yes", "no", "on", "off"):
+                return str(val).lower() in ("true", "1", "yes", "on")
+            return str(val)
+
     effective_type = _resolve_field_type(field_type)
     # bool
     if effective_type is bool:
@@ -113,6 +124,17 @@ def _convert_env_value_to_field_type(value: str, field_name: str, field_type: ty
         except (json.JSONDecodeError, TypeError):
             pass
         return tuple(elem_type(x.strip()) for x in str(val).split(",") if x.strip())
+    # For dataclass/complex types, try JSON then Python literal parsing to dict
+    try:
+        return json.loads(val)
+    except (json.JSONDecodeError, TypeError):
+        pass
+    try:
+        parsed = ast.literal_eval(val)
+        if isinstance(parsed, (dict, list)):
+            return parsed
+    except (ValueError, SyntaxError):
+        pass
     # Fallback: try int, float, then str
     try:
         return int(val)
@@ -330,6 +352,58 @@ def _sanitize_hf_overrides(hf_overrides: dict) -> dict | None:
     return result or None
 
 
+def _resolve_cached_model_path(model_name: str) -> str:
+    """Return a local snapshot path when the HF cache was stored with lowercase names.
+
+    Some model stores (e.g. RunPod pre-cached volumes) normalize repo IDs to
+    lowercase. HuggingFace Hub stores caches as
+    ``models--{org}--{model}/snapshots/{hash}/`` preserving the original casing,
+    so MODEL_NAME=Qwen/Qwen2.5-Coder-32B-Instruct-AWQ will miss a cache stored
+    as ``models--qwen--qwen2.5-coder-32b-instruct-awq/``.
+
+    If the exact-case cache directory is absent but a lowercase variant exists,
+    the latest snapshot path is returned so vLLM loads from disk rather than
+    attempting a redundant download.
+    """
+    if os.path.isabs(model_name):
+        return model_name
+
+    cache_dir = (
+        os.getenv("HUGGINGFACE_HUB_CACHE")
+        or os.getenv("HF_HOME")
+        or os.path.expanduser("~/.cache/huggingface/hub")
+    )
+
+    folder_name = f"models--{model_name.replace('/', '--')}"
+
+    if os.path.isdir(os.path.join(cache_dir, folder_name)):
+        return model_name
+
+    lower_dir = os.path.join(cache_dir, folder_name.lower())
+    if not os.path.isdir(lower_dir):
+        return model_name
+
+    snapshots_dir = os.path.join(lower_dir, "snapshots")
+    if not os.path.isdir(snapshots_dir):
+        return model_name
+
+    try:
+        snapshots = sorted(os.listdir(snapshots_dir))
+    except OSError:
+        return model_name
+
+    if not snapshots:
+        return model_name
+
+    resolved = os.path.join(snapshots_dir, snapshots[-1])
+    logging.info(
+        "MODEL_NAME %r not found at original casing in HF cache; "
+        "resolved to lowercase cached snapshot at %r",
+        model_name, resolved,
+    )
+    return resolved
+
+
 def get_local_args():
     """
     Retrieve local arguments from a JSON file.
@@ -401,6 +475,53 @@ def get_engine_args():
         if os.getenv("MAX_PARALLEL_LOADING_WORKERS"):
             logging.warning("Overriding MAX_PARALLEL_LOADING_WORKERS with None because more than 1 GPU is available.")
     
+    # LMCache requires HMA to be disabled
+    try:
+        _kv_transfer = args.get("kv_transfer_config")
+        if isinstance(_kv_transfer, str):
+            parsed = None
+            try:
+                parsed = json.loads(_kv_transfer)
+            except (json.JSONDecodeError, TypeError):
+                pass
+            if parsed is None:
+                try:
+                    result = ast.literal_eval(_kv_transfer)
+                    if isinstance(result, dict):
+                        parsed = result
+                except (ValueError, SyntaxError):
+                    pass
+            if parsed is not None:
+                _kv_transfer = parsed
+                args["kv_transfer_config"] = _kv_transfer
+        _kv_offload = args.get("kv_offloading_backend")
+
+        lmcache_via_offload = _kv_offload == "lmcache"
+        lmcache_via_transfer = (
+            isinstance(_kv_transfer, dict)
+            and isinstance(_kv_transfer.get("kv_connector"), str)
+            and "lmcache" in _kv_transfer.get("kv_connector", "").lower()
+        )
+        lmcache_detected = lmcache_via_offload or lmcache_via_transfer
+
+        if lmcache_detected:
+            current = args.get("disable_hybrid_kv_cache_manager")
+            if current is False:
+                logging.warning(
+                    "disable_hybrid_kv_cache_manager=False conflicts with LMCache; "
+                    "overriding to True (HMA must be disabled when using LMCache)"
+                )
+                args["disable_hybrid_kv_cache_manager"] = True
+            elif current is None:
+                args["disable_hybrid_kv_cache_manager"] = True
+                logging.info("LMCache detected: automatically setting disable_hybrid_kv_cache_manager=True")
+    except Exception as e:
+        logging.error(
+            "Failed to check LMCache configuration: %s",
+            e,
+            exc_info=True
+        )
+
     # Deprecated env args backwards compatibility
     if args.get("kv_cache_dtype") == "fp8_e5m2":
         args["kv_cache_dtype"] = "fp8"
@@ -458,4 +579,8 @@ def get_engine_args():
     if speculative_config:
         args["speculative_config"] = speculative_config
 
+    # Resolve lowercase HF cache paths (FDE-174)
+    if args.get("model"):
+        args["model"] = _resolve_cached_model_path(args["model"])
+
     return AsyncEngineArgs(**args)

src/start.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+if [ -n "${TRANSFORMERS_VERSION}" ]; then
+    echo "Installing transformers==${TRANSFORMERS_VERSION}"
+    uv pip install --system "transformers==${TRANSFORMERS_VERSION}"
+fi
+
+exec python3 /src/handler.py

src/tokenizer.py

@@ -1,10 +1,12 @@
-from transformers import AutoTokenizer
+import logging
 import os
 from typing import Union
 
+from transformers import AutoTokenizer
+
 class TokenizerWrapper:
     def __init__(self, tokenizer_name_or_path, tokenizer_revision, trust_remote_code):
-        print(f"tokenizer_name_or_path: {tokenizer_name_or_path}, tokenizer_revision: {tokenizer_revision}, trust_remote_code: {trust_remote_code}")
+        logging.debug("tokenizer_name_or_path: %s, tokenizer_revision: %s, trust_remote_code: %s", tokenizer_name_or_path, tokenizer_revision, trust_remote_code)
         self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, revision=tokenizer_revision or "main", trust_remote_code=trust_remote_code)
         self.custom_chat_template = os.getenv("CUSTOM_CHAT_TEMPLATE")
         self.has_chat_template = bool(self.tokenizer.chat_template) or bool(self.custom_chat_template)