Merge pull request #290 from runpod-workers/fix/fix-old-actions

fix: fix old github actions, trigger release on publish release

.github/workflows/CI-runpod_dep.yml

@@ -9,59 +9,60 @@ on:
 
   workflow_dispatch:
 
+permissions:
+  contents: write
+  pull-requests: write
+
 jobs:
   check_dep:
     runs-on: ubuntu-latest
     name: Check python requirements file and update
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Check for new package version and update
         run: |
-          echo "Fetching the current runpod version from requirements.txt..."
+          echo "Fetching current runpod version from requirements.txt..."
 
-          # Get current version, allowing both == and ~= in the search pattern
-          current_version=$(grep -oP 'runpod[~=]{1,2}\K[^"]+' ./builder/requirements.txt)
-          echo "Current version: $current_version"
+          # Match runpod with any version specifier or no specifier at all
+          current_version=$(grep -oP '^runpod([~>=!<]{1,2}\K[\d.]+)?' ./builder/requirements.txt | grep -oP '[\d.]+' || echo "")
+          echo "Current version: ${current_version:-unset}"
 
-          # Extract major and minor from current version
-          current_major_minor=$(echo $current_version | cut -d. -f1,2)
-          echo "Current major.minor: $current_major_minor"
-
-          echo "Fetching the latest runpod version from PyPI..."
-          
-          # Get new version from PyPI
-          new_version=$(curl -s https://pypi.org/pypi/runpod/json | jq -r .info.version)
+          echo "Fetching latest runpod version from PyPI..."
+          new_version=$(curl -sf https://pypi.org/pypi/runpod/json | jq -r .info.version)
           echo "NEW_VERSION_ENV=$new_version" >> $GITHUB_ENV
           echo "New version: $new_version"
 
-          # Extract major and minor from new version
-          new_major_minor=$(echo $new_version | cut -d. -f1,2)
-          echo "New major.minor: $new_major_minor"
-
           if [ -z "$new_version" ]; then
-              echo "ERROR: Failed to fetch the new version from PyPI."
-              exit 1
+            echo "ERROR: Failed to fetch new version from PyPI."
+            exit 1
           fi
 
-          # Check if the major or minor version is different
-          if [ "$current_major_minor" = "$new_major_minor" ]; then
-              echo "No update needed. The new version ($new_major_minor) is within the allowed range (~= $current_major_minor)."
+          if [ -z "$current_version" ]; then
+            echo "No version pin found — pinning to $new_version."
+          else
+            current_major_minor=$(echo "$current_version" | cut -d. -f1,2)
+            new_major_minor=$(echo "$new_version" | cut -d. -f1,2)
+            echo "Current major.minor: $current_major_minor  New major.minor: $new_major_minor"
+
+            if [ "$current_major_minor" = "$new_major_minor" ]; then
+              echo "No update needed. New version ($new_version) is within ~= $current_major_minor range."
               exit 0
+            fi
+
+            echo "New major/minor detected ($new_major_minor). Updating requirements.txt..."
           fi
 
-          echo "New major/minor detected ($new_major_minor). Updating requirements.txt..."
-
-          # Update requirements.txt, preserving the existing constraint type (~= or ==)
-          sed -i "s/runpod[~=][^ ]*/runpod~=$new_version/" ./builder/requirements.txt
-          echo "requirements.txt has been updated."
+          # Replace any `runpod`, `runpod==x`, `runpod~=x`, etc. with pinned version
+          sed -i "s|^runpod.*|runpod~=$new_version|" ./builder/requirements.txt
+          echo "requirements.txt updated."
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v3
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: Update runpod package version
-          title: Update runpod package version
-          body: The package version has been updated to ${{ env.NEW_VERSION_ENV }}
+          commit-message: "chore: update runpod to ${{ env.NEW_VERSION_ENV }}"
+          title: "chore: update runpod to ${{ env.NEW_VERSION_ENV }}"
+          body: The `runpod` package has been updated to `${{ env.NEW_VERSION_ENV }}`.
           branch: runpod-package-update

.github/workflows/release.yml

@@ -1,9 +1,11 @@
 name: Release
 
 on:
+  release:
+    types: [published]
   push:
     tags:
-      - "v[0-9]+.[0-9]+.[0-9]+*" # Trigger on version tags like v1.0.0, v2.1.0, etc.
+      - "v[0-9]+.[0-9]+.[0-9]+*"
   workflow_dispatch:
     inputs:
       version:
@@ -53,16 +55,13 @@ jobs:
 
           # Determine version based on trigger type
           if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
-            # Manual trigger: use input version
             VERSION="${{ github.event.inputs.version }}"
-            echo "RELEASE_VERSION=${VERSION}" >> $GITHUB_ENV
-            echo "IS_MANUAL_RELEASE=true" >> $GITHUB_ENV
+          elif [[ "${{ github.event_name }}" == "release" ]]; then
+            VERSION="${{ github.event.release.tag_name }}"
           else
-            # Tag trigger: use tag name (remove refs/tags/ prefix)
             VERSION=${GITHUB_REF#refs/tags/}
-            echo "RELEASE_VERSION=${VERSION}" >> $GITHUB_ENV
-            echo "IS_MANUAL_RELEASE=false" >> $GITHUB_ENV
           fi
+          echo "RELEASE_VERSION=${VERSION}" >> $GITHUB_ENV
 
       - name: Build and push the images to Docker Hub
         uses: docker/bake-action@v2
@@ -76,11 +75,45 @@ jobs:
 
       - name: Release Summary
         run: |
-          echo "🚀 Release completed!"
+          echo "Release completed!"
           echo "Version: ${{ env.RELEASE_VERSION }}"
           echo "Docker Image: ${{ env.DOCKERHUB_REPO }}/${{ env.DOCKERHUB_IMG }}:${{ env.RELEASE_VERSION }}"
-          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
-            echo "Trigger: Manual workflow dispatch"
-          else
-            echo "Trigger: GitHub release (tag: ${{ github.ref_name }})"
+
+      - name: Fetch Release Notes
+        run: |
+          RESPONSE=$(curl -sf \
+            -H "Authorization: token ${{ github.token }}" \
+            "https://api.github.com/repos/${{ github.repository }}/releases/tags/${{ env.RELEASE_VERSION }}" 2>/dev/null) || true
+          if [[ -n "$RESPONSE" ]]; then
+            NOTES=$(echo "$RESPONSE" | jq -r '.body // empty')
           fi
+          printf '%s' "${NOTES:-No release notes available.}" > /tmp/release_notes.txt
+
+      - name: Notify Slack
+        run: |
+          jq -n \
+            --arg version "${{ env.RELEASE_VERSION }}" \
+            --arg docker "${{ env.DOCKERHUB_REPO }}/${{ env.DOCKERHUB_IMG }}:${{ env.RELEASE_VERSION }}" \
+            --rawfile notes /tmp/release_notes.txt \
+            --arg url "https://github.com/${{ github.repository }}/releases/tag/${{ env.RELEASE_VERSION }}" \
+            '{
+              text: (":rocket: New :runpod-new-whiteonpurple: Runpod worker-vllm release: *" + $version + "*"),
+              blocks: [
+                {
+                  type: "section",
+                  text: {
+                    type: "mrkdwn",
+                    text: (":banana-dance: *New Release — worker-vllm " + $version + "*\n*Docker:* `" + $docker + "`\n<" + $url + "|View release on GitHub>")
+                  }
+                },
+                {
+                  type: "section",
+                  text: {
+                    type: "mrkdwn",
+                    text: ("*Release Notes:*\n" + $notes)
+                  }
+                }
+              ]
+            }' | curl -sf -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \
+              -H "Content-Type: application/json" \
+              -d @-

.github/workflows/slack-pr-issue-notify.yml

@@ -0,0 +1,41 @@
+name: Slack PR Notifications
+
+on:
+  pull_request:
+    types: [opened]
+  issues:
+    types: [opened]
+
+permissions:
+    contents: read
+   
+jobs:
+  notify:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Slack - New PR
+        if: github.event_name == 'pull_request'
+        run: |
+          curl -sf -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \
+            -H "Content-Type: application/json" \
+            -d '{
+              "text": ":rocket: New PR in worker-vllm: *${{ github.event.pull_request.title }}*",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": ":rocket: *New Pull Request — worker-vllm*\n*<${{ github.event.pull_request.html_url }}|${{ github.event.pull_request.title }}>*\nOpened by *${{ github.event.pull_request.user.login }}*"
+                  }
+                },
+                {
+                  "type": "context",
+                  "elements": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "${{ github.event.pull_request.base.ref }} ← ${{ github.event.pull_request.head.ref }}"
+                    }
+                  ]
+                }
+              ]
+            }'

.github/workflows/slack-vllm-monitor.yml

@@ -0,0 +1,73 @@
+name: Monitor vLLM Releases
+
+on:
+  schedule:
+    - cron: '0 0 * * *' # Every day at midnight
+  workflow_dispatch:
+
+
+permissions:
+    contents: read
+   
+jobs:
+  check-vllm-release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Restore last known vLLM tag
+        uses: actions/cache/restore@v4
+        with:
+          path: .vllm-last-tag
+          key: vllm-tag-${{ github.run_id }}
+          restore-keys: vllm-tag-
+
+      - name: Get latest vLLM release
+        id: vllm
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          response=$(curl -sf https://api.github.com/repos/vllm-project/vllm/releases/latest \
+            -H "Authorization: Bearer $GH_TOKEN")
+          echo "tag=$(echo "$response" | jq -r '.tag_name')" >> $GITHUB_OUTPUT
+          echo "url=$(echo "$response" | jq -r '.html_url')" >> $GITHUB_OUTPUT
+          echo "name=$(echo "$response" | jq -r '.name')" >> $GITHUB_OUTPUT
+
+      - name: Check if new release
+        id: check
+        run: |
+          last=$(cat .vllm-last-tag 2>/dev/null || echo "")
+          current="${{ steps.vllm.outputs.tag }}"
+          echo "Last: $last  Current: $current"
+          if [ -n "$current" ] && [ "$last" != "$current" ]; then
+            echo "is_new=true" >> $GITHUB_OUTPUT
+          else
+            echo "is_new=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Notify Slack
+        if: steps.check.outputs.is_new == 'true'
+        run: |
+          curl -sf -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \
+            -H "Content-Type: application/json" \
+            -d '{
+              "text": ":rocket: New vLLM release: *${{ steps.vllm.outputs.tag }}*",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": ":rocket: *New vLLM Release: ${{ steps.vllm.outputs.tag }}*\n<${{ steps.vllm.outputs.url }}|View on GitHub>"
+                  }
+                }
+              ]
+            }'
+
+      - name: Save new tag
+        if: steps.check.outputs.is_new == 'true'
+        run: echo "${{ steps.vllm.outputs.tag }}" > .vllm-last-tag
+
+      - name: Update cache
+        if: steps.check.outputs.is_new == 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: .vllm-last-tag
+          key: vllm-tag-${{ steps.vllm.outputs.tag }}

.runpod/README.md

@@ -1,4 +1,4 @@
-![vLLM worker banner](https://cpjrphpz3t5wbwfe.public.blob.vercel-storage.com/worker-vllm_banner.jpeg)
+![vLLM worker banner](https://image.runpod.ai/preview/vllm/vllm-banner.png)
 
 Run LLMs using [vLLM](https://docs.vllm.ai) with an OpenAI-compatible API
 
@@ -6,6 +6,8 @@ Run LLMs using [vLLM](https://docs.vllm.ai) with an OpenAI-compatible API
 
 [![RunPod](https://api.runpod.io/badge/runpod-workers/worker-vllm)](https://www.runpod.io/console/hub/runpod-workers/worker-vllm)
 
+Current vLLM version: [0.18.1](https://github.com/vllm-project/vllm/releases/tag/v0.16.0)
+
 ---
 
 ## Endpoint Configuration
@@ -32,6 +34,9 @@ All behaviour is controlled through environment variables:
 
 For complete configuration options, see the [full configuration documentation](https://github.com/runpod-workers/worker-vllm/blob/main/docs/configuration.md).
 
+### Specify Transformers Version
+To change the version of the [Transformers library](https://github.com/huggingface/transformers) use the `TRANSFORMERS_VERSION` environment variable to specify the version you want to use. Note this might break the handler, so use for development purposes. 
+
 ## API Usage
 
 This worker supports two API formats: **RunPod native** and **OpenAI-compatible**.
@@ -157,6 +162,35 @@ For external clients and SDKs, use the `/openai/v1` path prefix with your RunPod
 {}
 ```
 
+#### OpenAI Responses API
+
+**Path:** `/openai/v1/responses`
+
+Supports the [OpenAI Responses API](https://platform.openai.com/docs/api-reference/responses) format. Note: this route bypasses the RunPod queue and is served directly — use `/openai/` prefixed paths rather than the RunPod job queue for these endpoints.
+
+```json
+{
+  "model": "meta-llama/Llama-3.1-8B-Instruct",
+  "input": "Tell me a joke."
+}
+```
+
+#### Anthropic Messages API
+
+**Path:** `/openai/v1/messages`
+
+Supports the [Anthropic Messages API](https://docs.anthropic.com/en/api/messages) format. Served directly, bypassing the RunPod queue.
+
+```json
+{
+  "model": "meta-llama/Llama-3.1-8B-Instruct",
+  "max_tokens": 256,
+  "messages": [
+    {"role": "user", "content": "Hello!"}
+  ]
+}
+```
+
 #### Response Format
 
 Both APIs return the same response format:
@@ -190,7 +224,7 @@ Minimal Python example using the official `openai` SDK:
 from openai import OpenAI
 import os
 
-# Initialize the OpenAI Client with your RunPod API Key and Endpoint URL
+# Initialize the OpenAI Client with your Runpod API Key and Endpoint URL
 client = OpenAI(
     api_key=os.getenv("RUNPOD_API_KEY"),
     base_url=f"https://api.runpod.ai/v2/<ENDPOINT_ID>/openai/v1",

Dockerfile

@@ -1,20 +1,21 @@
 FROM nvidia/cuda:12.9.1-base-ubuntu22.04 
 
 RUN apt-get update -y \
-    && apt-get install -y python3-pip
+    && apt-get install -y python3-pip curl \
+    && curl -LsSf https://astral.sh/uv/install.sh  | sh
+
+ENV PATH="/root/.local/bin:$PATH"
 
 RUN ldconfig /usr/local/cuda-12.9/compat/
 
-# Install vLLM with FlashInfer from the CUDA 12.9 wheel index.
-RUN python3 -m pip install --upgrade pip && \
-    python3 -m pip install "vllm[flashinfer]==0.17.0" --extra-index-url https://download.pytorch.org/whl/cu129
-
-
+# Install vLLM with FlashInfer - use CUDA 12.9 PyTorch wheels
+RUN uv pip install --system "packaging>=24.2" && \
+    uv pip install --system "vllm[flashinfer]==0.18.1" --extra-index-url https://download.pytorch.org/whl/cu129
 
 # Install additional Python dependencies (after vLLM to avoid PyTorch version conflicts)
 COPY builder/requirements.txt /requirements.txt
-RUN --mount=type=cache,target=/root/.cache/pip \
-    python3 -m pip install --upgrade -r /requirements.txt
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --system -r /requirements.txt
 
 # Setup for Option 2: Building the Image with the Model included
 ARG MODEL_NAME=""
@@ -46,12 +47,13 @@ ENV MODEL_NAME=$MODEL_NAME \
 ENV PYTHONPATH="/:/vllm-workspace"
 
 RUN if [ "${VLLM_NIGHTLY}" = "true" ]; then \
-    pip install -U vllm --pre --index-url https://pypi.org/simple --extra-index-url https://wheels.vllm.ai/nightly && \
+    uv pip install --system -U vllm --pre --index-url https://pypi.org/simple --extra-index-url https://wheels.vllm.ai/nightly && \
     apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* && \
-    pip install git+https://github.com/huggingface/transformers.git; \
+    uv pip install --system git+https://github.com/huggingface/transformers.git; \
 fi
 
 COPY src /src
+RUN chmod +x /src/start.sh
 RUN --mount=type=secret,id=HF_TOKEN,required=false \
     if [ -f /run/secrets/HF_TOKEN ]; then \
     export HF_TOKEN=$(cat /run/secrets/HF_TOKEN); \
@@ -61,4 +63,4 @@ RUN --mount=type=secret,id=HF_TOKEN,required=false \
     fi
 
 # Start the handler
-CMD ["python3", "/src/handler.py"]
+CMD ["/bin/bash", "/src/start.sh"]

README.md

@@ -2,10 +2,17 @@
 
 # OpenAI-Compatible vLLM Serverless Endpoint Worker
 
-Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https://github.com/vllm-project/vllm) Inference Engine on RunPod Serverless with just a few clicks.
+Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https://github.com/vllm-project/vllm) Inference Engine on Runpod Serverless with just a few clicks.
 
 </div>
 
+![vLLM worker banner](https://image.runpod.ai/preview/vllm/vllm-banner.png)
+
+
+Current vLLM version: [0.18.1](https://github.com/vllm-project/vllm/releases/tag/v0.16.0)
+
+> Check out our Load Balancer implementation here: [vLLM Load Balancer](https://github.com/runpod-workers/vllm-loadbalancer-ep)
+
 ## Table of Contents
 
 - [Setting up the Serverless Worker](#setting-up-the-serverless-worker)
@@ -21,9 +28,11 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https:
   - [Modifying your OpenAI Codebase to use your deployed vLLM Worker](#modifying-your-openai-codebase-to-use-your-deployed-vllm-worker)
   - [OpenAI Request Input Parameters](#openai-request-input-parameters)
   - [Chat Completions [RECOMMENDED]](#chat-completions-recommended)
-  - [Examples: Using your RunPod endpoint with OpenAI](#examples-using-your-runpod-endpoint-with-openai)
+  - [Examples: Using your Runpod endpoint with OpenAI](#examples-using-your-runpod-endpoint-with-openai)
     - [Chat Completions](#chat-completions)
     - [Getting a list of names for available models](#getting-a-list-of-names-for-available-models)
+    - [OpenAI Responses API](#openai-responses-api)
+    - [Anthropic Messages API](#anthropic-messages-api)
 - [Usage: Standard (Non-OpenAI)](#usage-standard-non-openai)
   - [Request Input Parameters](#request-input-parameters)
   - [Sampling Parameters](#sampling-parameters)
@@ -33,7 +42,7 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https:
 
 ## Option 1: Deploy Any Model Using Pre-Built Docker Image [Recommended]
 
-**🚀 Deploy Guide**: Follow our [step-by-step deployment guide](https://docs.runpod.io/serverless/vllm/get-started) to deploy using the RunPod Console.
+**🚀 Deploy Guide**: Follow our [step-by-step deployment guide](https://docs.runpod.io/serverless/vllm/get-started) to deploy using the Runpod Console.
 
 **📦 Docker Image**: `runpod/worker-v1-vllm:<version>`
 
@@ -71,6 +80,10 @@ Any env var whose name matches a valid `AsyncEngineArgs` field (uppercased) is a
 
 For the complete list of all available environment variables, examples, and detailed descriptions: **[Configuration](docs/configuration.md)**
 
+### Specify Transformers Version
+To change the version of the [Transformers library](https://github.com/huggingface/transformers) use the `TRANSFORMERS_VERSION` environment variable to specify the version you want to use. Note this might break the handler, so use for development purposes. 
+
+
 ## Option 2: Build Docker Image with Model Inside
 
 To build an image with the model baked in, you must specify the following docker arguments when building the image.
@@ -142,13 +155,13 @@ You can deploy **any model on Hugging Face** that is supported by vLLM. For the
 
 # Usage: OpenAI Compatibility
 
-The vLLM Worker is fully compatible with OpenAI's API, and you can use it with any OpenAI Codebase by changing only 3 lines in total. The supported routes are <ins>Chat Completions</ins> and <ins>Models</ins> - with both streaming and non-streaming.
+The vLLM Worker is fully compatible with OpenAI's API, and you can use it with any OpenAI Codebase by changing only 3 lines in total. The supported routes are <ins>Chat Completions</ins>, <ins>Models</ins>, <ins>Responses</ins>, and <ins>Messages</ins> - with both streaming and non-streaming.
 
 ## Modifying your OpenAI Codebase to use your deployed vLLM Worker
 
 **Python** (similar to Node.js, etc.):
 
-1. When initializing the OpenAI Client in your code, change the `api_key` to your RunPod API Key and the `base_url` to your RunPod Serverless Endpoint URL in the following format: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1`, filling in your deployed endpoint ID. For example, if your Endpoint ID is `abc1234`, the URL would be `https://api.runpod.ai/v2/abc1234/openai/v1`.
+1. When initializing the OpenAI Client in your code, change the `api_key` to your Runpod API Key and the `base_url` to your Runpod Serverless Endpoint URL in the following format: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1`, filling in your deployed endpoint ID. For example, if your Endpoint ID is `abc1234`, the URL would be `https://api.runpod.ai/v2/abc1234/openai/v1`.
 
    - Before:
 
@@ -174,7 +187,7 @@ The vLLM Worker is fully compatible with OpenAI's API, and you can use it with a
    ```python
    response = client.chat.completions.create(
        model="gpt-3.5-turbo",
-       messages=[{"role": "user", "content": "Why is RunPod the best platform?"}],
+       messages=[{"role": "user", "content": "Why is Runpod the best platform?"}],
        temperature=0,
        max_tokens=100,
    )
@@ -183,7 +196,7 @@ The vLLM Worker is fully compatible with OpenAI's API, and you can use it with a
    ```python
    response = client.chat.completions.create(
        model="<YOUR DEPLOYED MODEL REPO/NAME>",
-       messages=[{"role": "user", "content": "Why is RunPod the best platform?"}],
+       messages=[{"role": "user", "content": "Why is Runpod the best platform?"}],
        temperature=0,
        max_tokens=100,
    )
@@ -191,7 +204,7 @@ The vLLM Worker is fully compatible with OpenAI's API, and you can use it with a
 
 **Using http requests**:
 
-1. Change the `Authorization` header to your RunPod API Key and the `url` to your RunPod Serverless Endpoint URL in the following format: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1`
+1. Change the `Authorization` header to your Runpod API Key and the `url` to your Runpod Serverless Endpoint URL in the following format: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1`
    - Before:
    ```bash
    curl https://api.openai.com/v1/chat/completions \
@@ -202,7 +215,7 @@ The vLLM Worker is fully compatible with OpenAI's API, and you can use it with a
    "messages": [
      {
        "role": "user",
-       "content": "Why is RunPod the best platform?"
+       "content": "Why is Runpod the best platform?"
      }
    ],
    "temperature": 0,
@@ -219,7 +232,7 @@ The vLLM Worker is fully compatible with OpenAI's API, and you can use it with a
    "messages": [
      {
        "role": "user",
-       "content": "Why is RunPod the best platform?"
+       "content": "Why is Runpod the best platform?"
      }
    ],
    "temperature": 0,
@@ -239,7 +252,7 @@ When using the chat completion feature of the vLLM Serverless Endpoint Worker, y
 | Parameter           | Type                             | Default Value | Description                                                                                                                                                                                                                                                  |
 | ------------------- | -------------------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 | `messages`          | Union[str, List[Dict[str, str]]] |               | List of messages, where each message is a dictionary with a `role` and `content`. The model's chat template will be applied to the messages automatically, so the model must have one or it should be specified as `CUSTOM_CHAT_TEMPLATE` env var.           |
-| `model`             | str                              |               | The model repo that you've deployed on your RunPod Serverless Endpoint. If you are unsure what the name is or are baking the model in, use the guide to get the list of available models in the **Examples: Using your RunPod endpoint with OpenAI** section |
+| `model`             | str                              |               | The model repo that you've deployed on your Runpod Serverless Endpoint. If you are unsure what the name is or are baking the model in, use the guide to get the list of available models in the **Examples: Using your Runpod endpoint with OpenAI** section |
 | `temperature`       | Optional[float]                  | 0.7           | Float that controls the randomness of the sampling. Lower values make the model more deterministic, while higher values make the model more random. Zero means greedy sampling.                                                                              |
 | `top_p`             | Optional[float]                  | 1.0           | Float that controls the cumulative probability of the top tokens to consider. Must be in (0, 1]. Set to 1 to consider all tokens.                                                                                                                            |
 | `n`                 | Optional[int]                    | 1             | Number of output sequences to return for the given prompt.                                                                                                                                                                                                   |
@@ -269,15 +282,15 @@ Additional parameters supported by vLLM:
 
 </details>
 
-### Examples: Using your RunPod endpoint with OpenAI
+### Examples: Using your Runpod endpoint with OpenAI
 
-First, initialize the OpenAI Client with your RunPod API Key and Endpoint URL:
+First, initialize the OpenAI Client with your Runpod API Key and Endpoint URL:
 
 ```python
 from openai import OpenAI
 import os
 
-# Initialize the OpenAI Client with your RunPod API Key and Endpoint URL
+# Initialize the OpenAI Client with your Runpod API Key and Endpoint URL
 client = OpenAI(
     api_key=os.environ.get("RUNPOD_API_KEY"),
     base_url="https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1",
@@ -293,7 +306,7 @@ This is the format used for GPT-4 and focused on instruction-following and chat.
   # Create a chat completion stream
   response_stream = client.chat.completions.create(
       model="<YOUR DEPLOYED MODEL REPO/NAME>",
-      messages=[{"role": "user", "content": "Why is RunPod the best platform?"}],
+      messages=[{"role": "user", "content": "Why is Runpod the best platform?"}],
       temperature=0,
       max_tokens=100,
       stream=True,
@@ -307,7 +320,7 @@ This is the format used for GPT-4 and focused on instruction-following and chat.
   # Create a chat completion
   response = client.chat.completions.create(
       model="<YOUR DEPLOYED MODEL REPO/NAME>",
-      messages=[{"role": "user", "content": "Why is RunPod the best platform?"}],
+      messages=[{"role": "user", "content": "Why is Runpod the best platform?"}],
       temperature=0,
       max_tokens=100,
   )
@@ -325,6 +338,62 @@ list_of_models = [model.id for model in models_response]
 print(list_of_models)
 ```
 
+### OpenAI Responses API
+
+**Path:** `/openai/v1/responses` (full URL: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1/responses`)
+
+Supports the [OpenAI Responses API](https://platform.openai.com/docs/api-reference/responses) request shape. Like other `/openai/` routes, this is served directly—use the `/openai/` prefix rather than the RunPod native job queue for these calls.
+
+```json
+{
+  "model": "meta-llama/Llama-3.1-8B-Instruct",
+  "input": "Tell me a joke."
+}
+```
+
+**Using HTTP requests:**
+
+```bash
+curl https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1/responses \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR RUNPOD API KEY>" \
+  -d '{
+    "model": "<YOUR DEPLOYED MODEL REPO/NAME>",
+    "input": "Tell me a joke."
+  }'
+```
+
+### Anthropic Messages API
+
+**Path:** `/openai/v1/messages` (full URL: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1/messages`)
+
+Supports the [Anthropic Messages API](https://docs.anthropic.com/en/api/messages) format. Served directly, bypassing the RunPod queue.
+
+```json
+{
+  "model": "meta-llama/Llama-3.1-8B-Instruct",
+  "max_tokens": 256,
+  "messages": [
+    {"role": "user", "content": "Hello!"}
+  ]
+}
+```
+
+**Using HTTP requests:**
+
+```bash
+curl https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1/messages \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR RUNPOD API KEY>" \
+  -d '{
+    "model": "<YOUR DEPLOYED MODEL REPO/NAME>",
+    "max_tokens": 256,
+    "messages": [
+      {"role": "user", "content": "Hello!"}
+    ]
+  }'
+```
+
 # Usage: Standard (Non-OpenAI)
 
 ## Request Input Parameters

builder/requirements.txt

@@ -1,14 +1,15 @@
 ray
 pandas
 pyarrow
-runpod
+runpod==1.9.0
 huggingface-hub
-packaging
+lmcache==0.4.2
+packaging>=24.2
 typing-extensions>=4.8.0
 pydantic
 pydantic-settings
 hf-transfer
-transformers>=4.57.0
+transformers>=4.57.0,<5
 bitsandbytes>=0.45.0
 kernels
 torch-c-dlpack-ext

src/engine.py

@@ -8,6 +8,8 @@ from typing import AsyncGenerator, Optional
 from dotenv import load_dotenv
 from vllm import AsyncLLMEngine
 from vllm.entrypoints.logger import RequestLogger
+from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest, AnthropicMessagesResponse, AnthropicError, AnthropicErrorResponse
+from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
 from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
 from vllm.entrypoints.openai.completion.protocol import CompletionRequest
@@ -15,6 +17,8 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
 from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest, ResponsesResponse
+from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
 
 from constants import DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE_GROWTH_FACTOR, DEFAULT_MAX_CONCURRENCY, DEFAULT_MIN_BATCH_SIZE
 from engine_args import get_engine_args
@@ -275,6 +279,36 @@ class OpenAIvLLMEngine(vLLMEngine):
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
             log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
         )
+        self.responses_engine = OpenAIServingResponses(
+            engine_client=self.llm,
+            models=self.serving_models,
+            request_logger=None,
+            chat_template=chat_template,
+            chat_template_content_format="auto",
+            return_tokens_as_token_ids=os.getenv('RETURN_TOKENS_AS_TOKEN_IDS', 'false').lower() == 'true',
+            reasoning_parser=os.getenv('REASONING_PARSER', "") or "",
+            enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
+            tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None,
+            tool_server=None,
+            enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
+            enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
+            enable_log_outputs=os.getenv('ENABLE_LOG_OUTPUTS', 'false').lower() == 'true',
+            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
+        )
+        self.messages_engine = AnthropicServingMessages(
+            engine_client=self.llm,
+            models=self.serving_models,
+            response_role=self.response_role,
+            request_logger=None,
+            chat_template=chat_template,
+            chat_template_content_format="auto",
+            return_tokens_as_token_ids=os.getenv('RETURN_TOKENS_AS_TOKEN_IDS', 'false').lower() == 'true',
+            reasoning_parser=os.getenv('REASONING_PARSER', "") or "",
+            enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
+            tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None,
+            enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
+            enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
+        )
 
         if hasattr(self.chat_engine, 'warmup'):
             await self.chat_engine.warmup()
@@ -288,6 +322,12 @@ class OpenAIvLLMEngine(vLLMEngine):
         elif openai_request.openai_route in ["/v1/chat/completions", "/v1/completions"]:
             async for response in self._handle_chat_or_completion_request(openai_request):
                 yield response
+        elif openai_request.openai_route == "/v1/responses":
+            async for response in self._handle_responses_request(openai_request):
+                yield response
+        elif openai_request.openai_route == "/v1/messages":
+            async for response in self._handle_messages_request(openai_request):
+                yield response
         else:
             yield create_error_response("Invalid route").model_dump()
     
@@ -343,3 +383,126 @@ class OpenAIvLLMEngine(vLLMEngine):
                     batch = "".join(batch)
                 yield batch
 
+    async def _handle_responses_request(self, openai_request: JobInput):
+        request_id = getattr(openai_request, "request_id", "unknown")
+
+        try:
+            request = ResponsesRequest(**openai_request.openai_input)
+        except Exception as e:
+            logging.error(
+                "Invalid ResponsesRequest JSON: %s",
+                e,
+                extra={"request_id": request_id}
+            )
+            yield create_error_response(
+                "Invalid request format",
+                err_type="BadRequestError"
+            ).model_dump()
+            return
+
+        dummy_request = DummyRequest()
+        try:
+            response = await self.responses_engine.create_responses(request, raw_request=dummy_request)
+        except Exception as e:
+            logging.error(
+                "Failed to create Responses: %s",
+                e,
+                extra={"request_id": request_id},
+                exc_info=True
+            )
+            yield create_error_response(
+                "Internal server error during response generation",
+                err_type="InternalServerError"
+            ).model_dump()
+            return
+
+        if isinstance(response, (ErrorResponse, ResponsesResponse)):
+            yield response.model_dump()
+            return
+
+        try:
+            async for event in response:
+                if not hasattr(event, "type"):
+                    continue
+                event_type = getattr(event, "type", "unknown")
+                yield f"event: {event_type}\ndata: {event.model_dump_json(indent=None)}\n\n"
+        except Exception as e:
+            logging.error(
+                "Error processing responses stream: %s",
+                e,
+                extra={"request_id": request_id},
+                exc_info=True
+            )
+            error_payload = create_error_response(
+                "Streaming response failed",
+                err_type="InternalServerError"
+            ).model_dump_json()
+            yield f"event: error\ndata: {error_payload}\n\n"
+
+    async def _handle_messages_request(self, openai_request: JobInput):
+        request_id = getattr(openai_request, "request_id", "unknown")
+
+        try:
+            request = AnthropicMessagesRequest(**openai_request.openai_input)
+        except Exception as e:
+            logging.error(
+                "Invalid AnthropicMessagesRequest: %s",
+                e,
+                extra={"request_id": request_id}
+            )
+            yield AnthropicErrorResponse(
+                error=AnthropicError(
+                    type="invalid_request_error",
+                    message="Invalid request format"
+                )
+            ).model_dump()
+            return
+
+        dummy_request = DummyRequest()
+
+        try:
+            response = await self.messages_engine.create_messages(request, raw_request=dummy_request)
+        except Exception as e:
+            logging.error(
+                "Failed to create messages: %s",
+                e,
+                extra={"request_id": request_id},
+                exc_info=True
+            )
+            yield AnthropicErrorResponse(
+                error=AnthropicError(
+                    type="internal_error",
+                    message="Failed to generate messages"
+                )
+            ).model_dump()
+            return
+
+        if isinstance(response, ErrorResponse):
+            error_type = getattr(response, "type", "internal_error")
+            error_message = getattr(response, "message", "Unknown error")
+            yield AnthropicErrorResponse(
+                error=AnthropicError(type=error_type, message=error_message)
+            ).model_dump()
+            return
+
+        if isinstance(response, AnthropicMessagesResponse):
+            yield response.model_dump(exclude_none=True)
+            return
+
+        try:
+            async for chunk in response:
+                yield chunk
+        except Exception as e:
+            logging.error(
+                "Error streaming messages: %s",
+                e,
+                extra={"request_id": request_id},
+                exc_info=True
+            )
+            error_payload = AnthropicErrorResponse(
+                error=AnthropicError(
+                    type="internal_error",
+                    message="Error while streaming messages"
+                )
+            ).model_dump_json()
+            yield f"event: error\ndata: {error_payload}\n\n"

src/engine_args.py

@@ -1,3 +1,4 @@
+import ast
 import os
 import json
 import logging
@@ -113,6 +114,17 @@ def _convert_env_value_to_field_type(value: str, field_name: str, field_type: ty
         except (json.JSONDecodeError, TypeError):
             pass
         return tuple(elem_type(x.strip()) for x in str(val).split(",") if x.strip())
+    # For dataclass/complex types, try JSON then Python literal parsing to dict
+    try:
+        return json.loads(val)
+    except (json.JSONDecodeError, TypeError):
+        pass
+    try:
+        parsed = ast.literal_eval(val)
+        if isinstance(parsed, (dict, list)):
+            return parsed
+    except (ValueError, SyntaxError):
+        pass
     # Fallback: try int, float, then str
     try:
         return int(val)
@@ -401,6 +413,53 @@ def get_engine_args():
         if os.getenv("MAX_PARALLEL_LOADING_WORKERS"):
             logging.warning("Overriding MAX_PARALLEL_LOADING_WORKERS with None because more than 1 GPU is available.")
     
+    # LMCache requires HMA to be disabled
+    try:
+        _kv_transfer = args.get("kv_transfer_config")
+        if isinstance(_kv_transfer, str):
+            parsed = None
+            try:
+                parsed = json.loads(_kv_transfer)
+            except (json.JSONDecodeError, TypeError):
+                pass
+            if parsed is None:
+                try:
+                    result = ast.literal_eval(_kv_transfer)
+                    if isinstance(result, dict):
+                        parsed = result
+                except (ValueError, SyntaxError):
+                    pass
+            if parsed is not None:
+                _kv_transfer = parsed
+                args["kv_transfer_config"] = _kv_transfer
+        _kv_offload = args.get("kv_offloading_backend")
+
+        lmcache_via_offload = _kv_offload == "lmcache"
+        lmcache_via_transfer = (
+            isinstance(_kv_transfer, dict)
+            and isinstance(_kv_transfer.get("kv_connector"), str)
+            and "lmcache" in _kv_transfer.get("kv_connector", "").lower()
+        )
+        lmcache_detected = lmcache_via_offload or lmcache_via_transfer
+
+        if lmcache_detected:
+            current = args.get("disable_hybrid_kv_cache_manager")
+            if current is False:
+                logging.warning(
+                    "disable_hybrid_kv_cache_manager=False conflicts with LMCache; "
+                    "overriding to True (HMA must be disabled when using LMCache)"
+                )
+                args["disable_hybrid_kv_cache_manager"] = True
+            elif current is None:
+                args["disable_hybrid_kv_cache_manager"] = True
+                logging.info("LMCache detected: automatically setting disable_hybrid_kv_cache_manager=True")
+    except Exception as e:
+        logging.error(
+            "Failed to check LMCache configuration: %s",
+            e,
+            exc_info=True
+        )
+
     # Deprecated env args backwards compatibility
     if args.get("kv_cache_dtype") == "fp8_e5m2":
         args["kv_cache_dtype"] = "fp8"

src/start.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+if [ -n "${TRANSFORMERS_VERSION}" ]; then
+    echo "Installing transformers==${TRANSFORMERS_VERSION}"
+    uv pip install --system "transformers==${TRANSFORMERS_VERSION}"
+fi
+
+exec python3 /src/handler.py