Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Owen Qwen
|
08c6ff7490 |
@@ -76,23 +76,11 @@ jobs:
|
||||
|
||||
- name: Release Summary
|
||||
run: |
|
||||
echo "Release completed!"
|
||||
echo "🚀 Release completed!"
|
||||
echo "Version: ${{ env.RELEASE_VERSION }}"
|
||||
echo "Docker Image: ${{ env.DOCKERHUB_REPO }}/${{ env.DOCKERHUB_IMG }}:${{ env.RELEASE_VERSION }}"
|
||||
|
||||
- name: Notify Slack
|
||||
run: |
|
||||
curl -sf -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"text": ":rocket: New :runpod-new-whiteonpurple: Runpod worker-vllm release: *${{ env.RELEASE_VERSION }}*",
|
||||
"blocks": [
|
||||
{
|
||||
"type": "section",
|
||||
"text": {
|
||||
"type": "mrkdwn",
|
||||
"text": ":banana-dance: *New Release — worker-vllm ${{ env.RELEASE_VERSION }}*\n*Docker:* `${{ env.DOCKERHUB_REPO }}/${{ env.DOCKERHUB_IMG }}:${{ env.RELEASE_VERSION }}`\n<https://github.com/${{ github.repository }}/releases/tag/${{ env.RELEASE_VERSION }}|View release on GitHub>"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
|
||||
echo "Trigger: Manual workflow dispatch"
|
||||
else
|
||||
echo "Trigger: GitHub release (tag: ${{ github.ref_name }})"
|
||||
fi
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
name: Slack PR Notifications
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened]
|
||||
issues:
|
||||
types: [opened]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
notify:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Notify Slack - New PR
|
||||
if: github.event_name == 'pull_request'
|
||||
run: |
|
||||
curl -sf -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"text": ":rocket: New PR in worker-vllm: *${{ github.event.pull_request.title }}*",
|
||||
"blocks": [
|
||||
{
|
||||
"type": "section",
|
||||
"text": {
|
||||
"type": "mrkdwn",
|
||||
"text": ":rocket: *New Pull Request — worker-vllm*\n*<${{ github.event.pull_request.html_url }}|${{ github.event.pull_request.title }}>*\nOpened by *${{ github.event.pull_request.user.login }}*"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "context",
|
||||
"elements": [
|
||||
{
|
||||
"type": "mrkdwn",
|
||||
"text": "${{ github.event.pull_request.base.ref }} ← ${{ github.event.pull_request.head.ref }}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}'
|
||||
@@ -1,73 +0,0 @@
|
||||
name: Monitor vLLM Releases
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 0 * * *' # Every day at midnight
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
check-vllm-release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Restore last known vLLM tag
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: .vllm-last-tag
|
||||
key: vllm-tag-${{ github.run_id }}
|
||||
restore-keys: vllm-tag-
|
||||
|
||||
- name: Get latest vLLM release
|
||||
id: vllm
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
response=$(curl -sf https://api.github.com/repos/vllm-project/vllm/releases/latest \
|
||||
-H "Authorization: Bearer $GH_TOKEN")
|
||||
echo "tag=$(echo "$response" | jq -r '.tag_name')" >> $GITHUB_OUTPUT
|
||||
echo "url=$(echo "$response" | jq -r '.html_url')" >> $GITHUB_OUTPUT
|
||||
echo "name=$(echo "$response" | jq -r '.name')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Check if new release
|
||||
id: check
|
||||
run: |
|
||||
last=$(cat .vllm-last-tag 2>/dev/null || echo "")
|
||||
current="${{ steps.vllm.outputs.tag }}"
|
||||
echo "Last: $last Current: $current"
|
||||
if [ -n "$current" ] && [ "$last" != "$current" ]; then
|
||||
echo "is_new=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "is_new=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Notify Slack
|
||||
if: steps.check.outputs.is_new == 'true'
|
||||
run: |
|
||||
curl -sf -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"text": ":rocket: New vLLM release: *${{ steps.vllm.outputs.tag }}*",
|
||||
"blocks": [
|
||||
{
|
||||
"type": "section",
|
||||
"text": {
|
||||
"type": "mrkdwn",
|
||||
"text": ":rocket: *New vLLM Release: ${{ steps.vllm.outputs.tag }}*\n<${{ steps.vllm.outputs.url }}|View on GitHub>"
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
|
||||
- name: Save new tag
|
||||
if: steps.check.outputs.is_new == 'true'
|
||||
run: echo "${{ steps.vllm.outputs.tag }}" > .vllm-last-tag
|
||||
|
||||
- name: Update cache
|
||||
if: steps.check.outputs.is_new == 'true'
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: .vllm-last-tag
|
||||
key: vllm-tag-${{ steps.vllm.outputs.tag }}
|
||||
+2
-34
@@ -1,4 +1,4 @@
|
||||
|
||||
|
||||
|
||||
Run LLMs using [vLLM](https://docs.vllm.ai) with an OpenAI-compatible API
|
||||
|
||||
@@ -32,9 +32,6 @@ All behaviour is controlled through environment variables:
|
||||
|
||||
For complete configuration options, see the [full configuration documentation](https://github.com/runpod-workers/worker-vllm/blob/main/docs/configuration.md).
|
||||
|
||||
### Specify Transformers Version
|
||||
To change the version of the [Transformers library](https://github.com/huggingface/transformers) use the `TRANSFORMERS_VERSION` environment variable to specify the version you want to use. Note this might break the handler, so use for development purposes.
|
||||
|
||||
## API Usage
|
||||
|
||||
This worker supports two API formats: **RunPod native** and **OpenAI-compatible**.
|
||||
@@ -160,35 +157,6 @@ For external clients and SDKs, use the `/openai/v1` path prefix with your RunPod
|
||||
{}
|
||||
```
|
||||
|
||||
#### OpenAI Responses API
|
||||
|
||||
**Path:** `/openai/v1/responses`
|
||||
|
||||
Supports the [OpenAI Responses API](https://platform.openai.com/docs/api-reference/responses) format. Note: this route bypasses the RunPod queue and is served directly — use `/openai/` prefixed paths rather than the RunPod job queue for these endpoints.
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "meta-llama/Llama-3.1-8B-Instruct",
|
||||
"input": "Tell me a joke."
|
||||
}
|
||||
```
|
||||
|
||||
#### Anthropic Messages API
|
||||
|
||||
**Path:** `/openai/v1/messages`
|
||||
|
||||
Supports the [Anthropic Messages API](https://docs.anthropic.com/en/api/messages) format. Served directly, bypassing the RunPod queue.
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "meta-llama/Llama-3.1-8B-Instruct",
|
||||
"max_tokens": 256,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello!"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### Response Format
|
||||
|
||||
Both APIs return the same response format:
|
||||
@@ -222,7 +190,7 @@ Minimal Python example using the official `openai` SDK:
|
||||
from openai import OpenAI
|
||||
import os
|
||||
|
||||
# Initialize the OpenAI Client with your Runpod API Key and Endpoint URL
|
||||
# Initialize the OpenAI Client with your RunPod API Key and Endpoint URL
|
||||
client = OpenAI(
|
||||
api_key=os.getenv("RUNPOD_API_KEY"),
|
||||
base_url=f"https://api.runpod.ai/v2/<ENDPOINT_ID>/openai/v1",
|
||||
|
||||
+11
-13
@@ -1,21 +1,20 @@
|
||||
FROM nvidia/cuda:12.9.1-base-ubuntu22.04
|
||||
|
||||
RUN apt-get update -y \
|
||||
&& apt-get install -y python3-pip curl \
|
||||
&& curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
|
||||
ENV PATH="/root/.local/bin:$PATH"
|
||||
&& apt-get install -y python3-pip
|
||||
|
||||
RUN ldconfig /usr/local/cuda-12.9/compat/
|
||||
|
||||
# Install vLLM with FlashInfer - use CUDA 12.9 PyTorch wheels
|
||||
RUN uv pip install --system "packaging>=24.2" && \
|
||||
uv pip install --system "vllm[flashinfer]==0.17.1" --extra-index-url https://download.pytorch.org/whl/cu129
|
||||
# Install vLLM with FlashInfer from the CUDA 12.9 wheel index.
|
||||
RUN python3 -m pip install --upgrade pip && \
|
||||
python3 -m pip install "vllm[flashinfer]==0.17.0" --extra-index-url https://download.pytorch.org/whl/cu129
|
||||
|
||||
|
||||
|
||||
# Install additional Python dependencies (after vLLM to avoid PyTorch version conflicts)
|
||||
COPY builder/requirements.txt /requirements.txt
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip install --system -r /requirements.txt
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
python3 -m pip install --upgrade -r /requirements.txt
|
||||
|
||||
# Setup for Option 2: Building the Image with the Model included
|
||||
ARG MODEL_NAME=""
|
||||
@@ -47,13 +46,12 @@ ENV MODEL_NAME=$MODEL_NAME \
|
||||
ENV PYTHONPATH="/:/vllm-workspace"
|
||||
|
||||
RUN if [ "${VLLM_NIGHTLY}" = "true" ]; then \
|
||||
uv pip install --system -U vllm --pre --index-url https://pypi.org/simple --extra-index-url https://wheels.vllm.ai/nightly && \
|
||||
pip install -U vllm --pre --index-url https://pypi.org/simple --extra-index-url https://wheels.vllm.ai/nightly && \
|
||||
apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* && \
|
||||
uv pip install --system git+https://github.com/huggingface/transformers.git; \
|
||||
pip install git+https://github.com/huggingface/transformers.git; \
|
||||
fi
|
||||
|
||||
COPY src /src
|
||||
RUN chmod +x /src/start.sh
|
||||
RUN --mount=type=secret,id=HF_TOKEN,required=false \
|
||||
if [ -f /run/secrets/HF_TOKEN ]; then \
|
||||
export HF_TOKEN=$(cat /run/secrets/HF_TOKEN); \
|
||||
@@ -63,4 +61,4 @@ RUN --mount=type=secret,id=HF_TOKEN,required=false \
|
||||
fi
|
||||
|
||||
# Start the handler
|
||||
CMD ["/bin/bash", "/src/start.sh"]
|
||||
CMD ["python3", "/src/handler.py"]
|
||||
|
||||
@@ -2,16 +2,10 @@
|
||||
|
||||
# OpenAI-Compatible vLLM Serverless Endpoint Worker
|
||||
|
||||
Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https://github.com/vllm-project/vllm) Inference Engine on Runpod Serverless with just a few clicks.
|
||||
Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https://github.com/vllm-project/vllm) Inference Engine on RunPod Serverless with just a few clicks.
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
Current vLLM version: [0.16.0](https://github.com/vllm-project/vllm/releases/tag/v0.16.0)
|
||||
|
||||
> Check out our Load Balancer implementation here: [vLLM Load Balancer](https://github.com/runpod-workers/vllm-loadbalancer-ep)
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Setting up the Serverless Worker](#setting-up-the-serverless-worker)
|
||||
@@ -27,11 +21,9 @@ Current vLLM version: [0.16.0](https://github.com/vllm-project/vllm/releases/tag
|
||||
- [Modifying your OpenAI Codebase to use your deployed vLLM Worker](#modifying-your-openai-codebase-to-use-your-deployed-vllm-worker)
|
||||
- [OpenAI Request Input Parameters](#openai-request-input-parameters)
|
||||
- [Chat Completions [RECOMMENDED]](#chat-completions-recommended)
|
||||
- [Examples: Using your Runpod endpoint with OpenAI](#examples-using-your-runpod-endpoint-with-openai)
|
||||
- [Examples: Using your RunPod endpoint with OpenAI](#examples-using-your-runpod-endpoint-with-openai)
|
||||
- [Chat Completions](#chat-completions)
|
||||
- [Getting a list of names for available models](#getting-a-list-of-names-for-available-models)
|
||||
- [OpenAI Responses API](#openai-responses-api)
|
||||
- [Anthropic Messages API](#anthropic-messages-api)
|
||||
- [Usage: Standard (Non-OpenAI)](#usage-standard-non-openai)
|
||||
- [Request Input Parameters](#request-input-parameters)
|
||||
- [Sampling Parameters](#sampling-parameters)
|
||||
@@ -41,7 +33,7 @@ Current vLLM version: [0.16.0](https://github.com/vllm-project/vllm/releases/tag
|
||||
|
||||
## Option 1: Deploy Any Model Using Pre-Built Docker Image [Recommended]
|
||||
|
||||
**🚀 Deploy Guide**: Follow our [step-by-step deployment guide](https://docs.runpod.io/serverless/vllm/get-started) to deploy using the Runpod Console.
|
||||
**🚀 Deploy Guide**: Follow our [step-by-step deployment guide](https://docs.runpod.io/serverless/vllm/get-started) to deploy using the RunPod Console.
|
||||
|
||||
**📦 Docker Image**: `runpod/worker-v1-vllm:<version>`
|
||||
|
||||
@@ -79,10 +71,6 @@ Any env var whose name matches a valid `AsyncEngineArgs` field (uppercased) is a
|
||||
|
||||
For the complete list of all available environment variables, examples, and detailed descriptions: **[Configuration](docs/configuration.md)**
|
||||
|
||||
### Specify Transformers Version
|
||||
To change the version of the [Transformers library](https://github.com/huggingface/transformers) use the `TRANSFORMERS_VERSION` environment variable to specify the version you want to use. Note this might break the handler, so use for development purposes.
|
||||
|
||||
|
||||
## Option 2: Build Docker Image with Model Inside
|
||||
|
||||
To build an image with the model baked in, you must specify the following docker arguments when building the image.
|
||||
@@ -154,13 +142,13 @@ You can deploy **any model on Hugging Face** that is supported by vLLM. For the
|
||||
|
||||
# Usage: OpenAI Compatibility
|
||||
|
||||
The vLLM Worker is fully compatible with OpenAI's API, and you can use it with any OpenAI Codebase by changing only 3 lines in total. The supported routes are <ins>Chat Completions</ins>, <ins>Models</ins>, <ins>Responses</ins>, and <ins>Messages</ins> - with both streaming and non-streaming.
|
||||
The vLLM Worker is fully compatible with OpenAI's API, and you can use it with any OpenAI Codebase by changing only 3 lines in total. The supported routes are <ins>Chat Completions</ins> and <ins>Models</ins> - with both streaming and non-streaming.
|
||||
|
||||
## Modifying your OpenAI Codebase to use your deployed vLLM Worker
|
||||
|
||||
**Python** (similar to Node.js, etc.):
|
||||
|
||||
1. When initializing the OpenAI Client in your code, change the `api_key` to your Runpod API Key and the `base_url` to your Runpod Serverless Endpoint URL in the following format: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1`, filling in your deployed endpoint ID. For example, if your Endpoint ID is `abc1234`, the URL would be `https://api.runpod.ai/v2/abc1234/openai/v1`.
|
||||
1. When initializing the OpenAI Client in your code, change the `api_key` to your RunPod API Key and the `base_url` to your RunPod Serverless Endpoint URL in the following format: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1`, filling in your deployed endpoint ID. For example, if your Endpoint ID is `abc1234`, the URL would be `https://api.runpod.ai/v2/abc1234/openai/v1`.
|
||||
|
||||
- Before:
|
||||
|
||||
@@ -186,7 +174,7 @@ The vLLM Worker is fully compatible with OpenAI's API, and you can use it with a
|
||||
```python
|
||||
response = client.chat.completions.create(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "Why is Runpod the best platform?"}],
|
||||
messages=[{"role": "user", "content": "Why is RunPod the best platform?"}],
|
||||
temperature=0,
|
||||
max_tokens=100,
|
||||
)
|
||||
@@ -195,7 +183,7 @@ The vLLM Worker is fully compatible with OpenAI's API, and you can use it with a
|
||||
```python
|
||||
response = client.chat.completions.create(
|
||||
model="<YOUR DEPLOYED MODEL REPO/NAME>",
|
||||
messages=[{"role": "user", "content": "Why is Runpod the best platform?"}],
|
||||
messages=[{"role": "user", "content": "Why is RunPod the best platform?"}],
|
||||
temperature=0,
|
||||
max_tokens=100,
|
||||
)
|
||||
@@ -203,7 +191,7 @@ The vLLM Worker is fully compatible with OpenAI's API, and you can use it with a
|
||||
|
||||
**Using http requests**:
|
||||
|
||||
1. Change the `Authorization` header to your Runpod API Key and the `url` to your Runpod Serverless Endpoint URL in the following format: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1`
|
||||
1. Change the `Authorization` header to your RunPod API Key and the `url` to your RunPod Serverless Endpoint URL in the following format: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1`
|
||||
- Before:
|
||||
```bash
|
||||
curl https://api.openai.com/v1/chat/completions \
|
||||
@@ -214,7 +202,7 @@ The vLLM Worker is fully compatible with OpenAI's API, and you can use it with a
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Why is Runpod the best platform?"
|
||||
"content": "Why is RunPod the best platform?"
|
||||
}
|
||||
],
|
||||
"temperature": 0,
|
||||
@@ -231,7 +219,7 @@ The vLLM Worker is fully compatible with OpenAI's API, and you can use it with a
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Why is Runpod the best platform?"
|
||||
"content": "Why is RunPod the best platform?"
|
||||
}
|
||||
],
|
||||
"temperature": 0,
|
||||
@@ -251,7 +239,7 @@ When using the chat completion feature of the vLLM Serverless Endpoint Worker, y
|
||||
| Parameter | Type | Default Value | Description |
|
||||
| ------------------- | -------------------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `messages` | Union[str, List[Dict[str, str]]] | | List of messages, where each message is a dictionary with a `role` and `content`. The model's chat template will be applied to the messages automatically, so the model must have one or it should be specified as `CUSTOM_CHAT_TEMPLATE` env var. |
|
||||
| `model` | str | | The model repo that you've deployed on your Runpod Serverless Endpoint. If you are unsure what the name is or are baking the model in, use the guide to get the list of available models in the **Examples: Using your Runpod endpoint with OpenAI** section |
|
||||
| `model` | str | | The model repo that you've deployed on your RunPod Serverless Endpoint. If you are unsure what the name is or are baking the model in, use the guide to get the list of available models in the **Examples: Using your RunPod endpoint with OpenAI** section |
|
||||
| `temperature` | Optional[float] | 0.7 | Float that controls the randomness of the sampling. Lower values make the model more deterministic, while higher values make the model more random. Zero means greedy sampling. |
|
||||
| `top_p` | Optional[float] | 1.0 | Float that controls the cumulative probability of the top tokens to consider. Must be in (0, 1]. Set to 1 to consider all tokens. |
|
||||
| `n` | Optional[int] | 1 | Number of output sequences to return for the given prompt. |
|
||||
@@ -281,15 +269,15 @@ Additional parameters supported by vLLM:
|
||||
|
||||
</details>
|
||||
|
||||
### Examples: Using your Runpod endpoint with OpenAI
|
||||
### Examples: Using your RunPod endpoint with OpenAI
|
||||
|
||||
First, initialize the OpenAI Client with your Runpod API Key and Endpoint URL:
|
||||
First, initialize the OpenAI Client with your RunPod API Key and Endpoint URL:
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
import os
|
||||
|
||||
# Initialize the OpenAI Client with your Runpod API Key and Endpoint URL
|
||||
# Initialize the OpenAI Client with your RunPod API Key and Endpoint URL
|
||||
client = OpenAI(
|
||||
api_key=os.environ.get("RUNPOD_API_KEY"),
|
||||
base_url="https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1",
|
||||
@@ -305,7 +293,7 @@ This is the format used for GPT-4 and focused on instruction-following and chat.
|
||||
# Create a chat completion stream
|
||||
response_stream = client.chat.completions.create(
|
||||
model="<YOUR DEPLOYED MODEL REPO/NAME>",
|
||||
messages=[{"role": "user", "content": "Why is Runpod the best platform?"}],
|
||||
messages=[{"role": "user", "content": "Why is RunPod the best platform?"}],
|
||||
temperature=0,
|
||||
max_tokens=100,
|
||||
stream=True,
|
||||
@@ -319,7 +307,7 @@ This is the format used for GPT-4 and focused on instruction-following and chat.
|
||||
# Create a chat completion
|
||||
response = client.chat.completions.create(
|
||||
model="<YOUR DEPLOYED MODEL REPO/NAME>",
|
||||
messages=[{"role": "user", "content": "Why is Runpod the best platform?"}],
|
||||
messages=[{"role": "user", "content": "Why is RunPod the best platform?"}],
|
||||
temperature=0,
|
||||
max_tokens=100,
|
||||
)
|
||||
@@ -337,62 +325,6 @@ list_of_models = [model.id for model in models_response]
|
||||
print(list_of_models)
|
||||
```
|
||||
|
||||
### OpenAI Responses API
|
||||
|
||||
**Path:** `/openai/v1/responses` (full URL: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1/responses`)
|
||||
|
||||
Supports the [OpenAI Responses API](https://platform.openai.com/docs/api-reference/responses) request shape. Like other `/openai/` routes, this is served directly—use the `/openai/` prefix rather than the RunPod native job queue for these calls.
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "meta-llama/Llama-3.1-8B-Instruct",
|
||||
"input": "Tell me a joke."
|
||||
}
|
||||
```
|
||||
|
||||
**Using HTTP requests:**
|
||||
|
||||
```bash
|
||||
curl https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1/responses \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer <YOUR RUNPOD API KEY>" \
|
||||
-d '{
|
||||
"model": "<YOUR DEPLOYED MODEL REPO/NAME>",
|
||||
"input": "Tell me a joke."
|
||||
}'
|
||||
```
|
||||
|
||||
### Anthropic Messages API
|
||||
|
||||
**Path:** `/openai/v1/messages` (full URL: `https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1/messages`)
|
||||
|
||||
Supports the [Anthropic Messages API](https://docs.anthropic.com/en/api/messages) format. Served directly, bypassing the RunPod queue.
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "meta-llama/Llama-3.1-8B-Instruct",
|
||||
"max_tokens": 256,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello!"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Using HTTP requests:**
|
||||
|
||||
```bash
|
||||
curl https://api.runpod.ai/v2/<YOUR ENDPOINT ID>/openai/v1/messages \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer <YOUR RUNPOD API KEY>" \
|
||||
-d '{
|
||||
"model": "<YOUR DEPLOYED MODEL REPO/NAME>",
|
||||
"max_tokens": 256,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello!"}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
# Usage: Standard (Non-OpenAI)
|
||||
|
||||
## Request Input Parameters
|
||||
|
||||
@@ -3,13 +3,12 @@ pandas
|
||||
pyarrow
|
||||
runpod
|
||||
huggingface-hub
|
||||
lmcache==0.4.2
|
||||
packaging>=24.2
|
||||
packaging
|
||||
typing-extensions>=4.8.0
|
||||
pydantic
|
||||
pydantic-settings
|
||||
hf-transfer
|
||||
transformers>=4.57.0,<5
|
||||
transformers>=4.57.0
|
||||
bitsandbytes>=0.45.0
|
||||
kernels
|
||||
torch-c-dlpack-ext
|
||||
|
||||
-163
@@ -8,8 +8,6 @@ from typing import AsyncGenerator, Optional
|
||||
from dotenv import load_dotenv
|
||||
from vllm import AsyncLLMEngine
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest, AnthropicMessagesResponse, AnthropicError, AnthropicErrorResponse
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
@@ -17,8 +15,6 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest, ResponsesResponse
|
||||
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
||||
|
||||
from constants import DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE_GROWTH_FACTOR, DEFAULT_MAX_CONCURRENCY, DEFAULT_MIN_BATCH_SIZE
|
||||
from engine_args import get_engine_args
|
||||
@@ -279,36 +275,6 @@ class OpenAIvLLMEngine(vLLMEngine):
|
||||
enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
|
||||
log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
|
||||
)
|
||||
self.responses_engine = OpenAIServingResponses(
|
||||
engine_client=self.llm,
|
||||
models=self.serving_models,
|
||||
request_logger=None,
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format="auto",
|
||||
return_tokens_as_token_ids=os.getenv('RETURN_TOKENS_AS_TOKEN_IDS', 'false').lower() == 'true',
|
||||
reasoning_parser=os.getenv('REASONING_PARSER', "") or "",
|
||||
enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
|
||||
tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None,
|
||||
tool_server=None,
|
||||
enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
|
||||
enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
|
||||
enable_log_outputs=os.getenv('ENABLE_LOG_OUTPUTS', 'false').lower() == 'true',
|
||||
log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
|
||||
)
|
||||
self.messages_engine = AnthropicServingMessages(
|
||||
engine_client=self.llm,
|
||||
models=self.serving_models,
|
||||
response_role=self.response_role,
|
||||
request_logger=None,
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format="auto",
|
||||
return_tokens_as_token_ids=os.getenv('RETURN_TOKENS_AS_TOKEN_IDS', 'false').lower() == 'true',
|
||||
reasoning_parser=os.getenv('REASONING_PARSER', "") or "",
|
||||
enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
|
||||
tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None,
|
||||
enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
|
||||
enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
|
||||
)
|
||||
|
||||
if hasattr(self.chat_engine, 'warmup'):
|
||||
await self.chat_engine.warmup()
|
||||
@@ -322,12 +288,6 @@ class OpenAIvLLMEngine(vLLMEngine):
|
||||
elif openai_request.openai_route in ["/v1/chat/completions", "/v1/completions"]:
|
||||
async for response in self._handle_chat_or_completion_request(openai_request):
|
||||
yield response
|
||||
elif openai_request.openai_route == "/v1/responses":
|
||||
async for response in self._handle_responses_request(openai_request):
|
||||
yield response
|
||||
elif openai_request.openai_route == "/v1/messages":
|
||||
async for response in self._handle_messages_request(openai_request):
|
||||
yield response
|
||||
else:
|
||||
yield create_error_response("Invalid route").model_dump()
|
||||
|
||||
@@ -383,126 +343,3 @@ class OpenAIvLLMEngine(vLLMEngine):
|
||||
batch = "".join(batch)
|
||||
yield batch
|
||||
|
||||
async def _handle_responses_request(self, openai_request: JobInput):
|
||||
request_id = getattr(openai_request, "request_id", "unknown")
|
||||
|
||||
try:
|
||||
request = ResponsesRequest(**openai_request.openai_input)
|
||||
except Exception as e:
|
||||
logging.error(
|
||||
"Invalid ResponsesRequest JSON: %s",
|
||||
e,
|
||||
extra={"request_id": request_id}
|
||||
)
|
||||
yield create_error_response(
|
||||
"Invalid request format",
|
||||
err_type="BadRequestError"
|
||||
).model_dump()
|
||||
return
|
||||
|
||||
dummy_request = DummyRequest()
|
||||
try:
|
||||
response = await self.responses_engine.create_responses(request, raw_request=dummy_request)
|
||||
except Exception as e:
|
||||
logging.error(
|
||||
"Failed to create Responses: %s",
|
||||
e,
|
||||
extra={"request_id": request_id},
|
||||
exc_info=True
|
||||
)
|
||||
yield create_error_response(
|
||||
"Internal server error during response generation",
|
||||
err_type="InternalServerError"
|
||||
).model_dump()
|
||||
return
|
||||
|
||||
if isinstance(response, (ErrorResponse, ResponsesResponse)):
|
||||
yield response.model_dump()
|
||||
return
|
||||
|
||||
try:
|
||||
async for event in response:
|
||||
if not hasattr(event, "type"):
|
||||
continue
|
||||
event_type = getattr(event, "type", "unknown")
|
||||
yield f"event: {event_type}\ndata: {event.model_dump_json(indent=None)}\n\n"
|
||||
except Exception as e:
|
||||
logging.error(
|
||||
"Error processing responses stream: %s",
|
||||
e,
|
||||
extra={"request_id": request_id},
|
||||
exc_info=True
|
||||
)
|
||||
error_payload = create_error_response(
|
||||
"Streaming response failed",
|
||||
err_type="InternalServerError"
|
||||
).model_dump_json()
|
||||
yield f"event: error\ndata: {error_payload}\n\n"
|
||||
|
||||
async def _handle_messages_request(self, openai_request: JobInput):
|
||||
request_id = getattr(openai_request, "request_id", "unknown")
|
||||
|
||||
try:
|
||||
request = AnthropicMessagesRequest(**openai_request.openai_input)
|
||||
except Exception as e:
|
||||
logging.error(
|
||||
"Invalid AnthropicMessagesRequest: %s",
|
||||
e,
|
||||
extra={"request_id": request_id}
|
||||
)
|
||||
yield AnthropicErrorResponse(
|
||||
error=AnthropicError(
|
||||
type="invalid_request_error",
|
||||
message="Invalid request format"
|
||||
)
|
||||
).model_dump()
|
||||
return
|
||||
|
||||
dummy_request = DummyRequest()
|
||||
|
||||
try:
|
||||
response = await self.messages_engine.create_messages(request, raw_request=dummy_request)
|
||||
except Exception as e:
|
||||
logging.error(
|
||||
"Failed to create messages: %s",
|
||||
e,
|
||||
extra={"request_id": request_id},
|
||||
exc_info=True
|
||||
)
|
||||
yield AnthropicErrorResponse(
|
||||
error=AnthropicError(
|
||||
type="internal_error",
|
||||
message="Failed to generate messages"
|
||||
)
|
||||
).model_dump()
|
||||
return
|
||||
|
||||
if isinstance(response, ErrorResponse):
|
||||
error_type = getattr(response, "type", "internal_error")
|
||||
error_message = getattr(response, "message", "Unknown error")
|
||||
yield AnthropicErrorResponse(
|
||||
error=AnthropicError(type=error_type, message=error_message)
|
||||
).model_dump()
|
||||
return
|
||||
|
||||
if isinstance(response, AnthropicMessagesResponse):
|
||||
yield response.model_dump(exclude_none=True)
|
||||
return
|
||||
|
||||
try:
|
||||
async for chunk in response:
|
||||
yield chunk
|
||||
except Exception as e:
|
||||
logging.error(
|
||||
"Error streaming messages: %s",
|
||||
e,
|
||||
extra={"request_id": request_id},
|
||||
exc_info=True
|
||||
)
|
||||
error_payload = AnthropicErrorResponse(
|
||||
error=AnthropicError(
|
||||
type="internal_error",
|
||||
message="Error while streaming messages"
|
||||
)
|
||||
).model_dump_json()
|
||||
yield f"event: error\ndata: {error_payload}\n\n"
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import ast
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
@@ -114,17 +113,6 @@ def _convert_env_value_to_field_type(value: str, field_name: str, field_type: ty
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
return tuple(elem_type(x.strip()) for x in str(val).split(",") if x.strip())
|
||||
# For dataclass/complex types, try JSON then Python literal parsing to dict
|
||||
try:
|
||||
return json.loads(val)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
try:
|
||||
parsed = ast.literal_eval(val)
|
||||
if isinstance(parsed, (dict, list)):
|
||||
return parsed
|
||||
except (ValueError, SyntaxError):
|
||||
pass
|
||||
# Fallback: try int, float, then str
|
||||
try:
|
||||
return int(val)
|
||||
@@ -413,53 +401,6 @@ def get_engine_args():
|
||||
if os.getenv("MAX_PARALLEL_LOADING_WORKERS"):
|
||||
logging.warning("Overriding MAX_PARALLEL_LOADING_WORKERS with None because more than 1 GPU is available.")
|
||||
|
||||
# LMCache requires HMA to be disabled
|
||||
try:
|
||||
_kv_transfer = args.get("kv_transfer_config")
|
||||
if isinstance(_kv_transfer, str):
|
||||
parsed = None
|
||||
try:
|
||||
parsed = json.loads(_kv_transfer)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
if parsed is None:
|
||||
try:
|
||||
result = ast.literal_eval(_kv_transfer)
|
||||
if isinstance(result, dict):
|
||||
parsed = result
|
||||
except (ValueError, SyntaxError):
|
||||
pass
|
||||
if parsed is not None:
|
||||
_kv_transfer = parsed
|
||||
args["kv_transfer_config"] = _kv_transfer
|
||||
_kv_offload = args.get("kv_offloading_backend")
|
||||
|
||||
lmcache_via_offload = _kv_offload == "lmcache"
|
||||
lmcache_via_transfer = (
|
||||
isinstance(_kv_transfer, dict)
|
||||
and isinstance(_kv_transfer.get("kv_connector"), str)
|
||||
and "lmcache" in _kv_transfer.get("kv_connector", "").lower()
|
||||
)
|
||||
lmcache_detected = lmcache_via_offload or lmcache_via_transfer
|
||||
|
||||
if lmcache_detected:
|
||||
current = args.get("disable_hybrid_kv_cache_manager")
|
||||
if current is False:
|
||||
logging.warning(
|
||||
"disable_hybrid_kv_cache_manager=False conflicts with LMCache; "
|
||||
"overriding to True (HMA must be disabled when using LMCache)"
|
||||
)
|
||||
args["disable_hybrid_kv_cache_manager"] = True
|
||||
elif current is None:
|
||||
args["disable_hybrid_kv_cache_manager"] = True
|
||||
logging.info("LMCache detected: automatically setting disable_hybrid_kv_cache_manager=True")
|
||||
except Exception as e:
|
||||
logging.error(
|
||||
"Failed to check LMCache configuration: %s",
|
||||
e,
|
||||
exc_info=True
|
||||
)
|
||||
|
||||
# Deprecated env args backwards compatibility
|
||||
if args.get("kv_cache_dtype") == "fp8_e5m2":
|
||||
args["kv_cache_dtype"] = "fp8"
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
if [ -n "${TRANSFORMERS_VERSION}" ]; then
|
||||
echo "Installing transformers==${TRANSFORMERS_VERSION}"
|
||||
uv pip install --system "transformers==${TRANSFORMERS_VERSION}"
|
||||
fi
|
||||
|
||||
exec python3 /src/handler.py
|
||||
Reference in New Issue
Block a user