From ba19cc97bfd276b888b6064b34dfef8fb1452961 Mon Sep 17 00:00:00 2001 From: pandyamarut Date: Thu, 8 May 2025 11:34:58 -0700 Subject: [PATCH] update vllm Signed-off-by: pandyamarut --- README.md | 6 ++--- worker-config.json | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7821683..91fc0fe 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https: ### 1. UI for Deploying vLLM Worker on RunPod console: ![Demo of Deploying vLLM Worker on RunPod console with new UI](media/ui_demo.gif) -### 2. Worker vLLM `v2.4.0` with vLLM `0.8.4` now available under `stable` tags +### 2. Worker vLLM `v2.5.0` with vLLM `0.8.5` now available under `stable` tags -Update v2.3.0 is now available, use the image tag `runpod/worker-v1-vllm:v2.4.0stable-cuda12.1.0`. +Update v2.5.0 is now available, use the image tag `runpod/worker-v1-vllm:v2.5.0stable-cuda12.1.0`. ### 3. OpenAI-Compatible [Embedding Worker](https://github.com/runpod-workers/worker-infinity-embedding) Released Deploy your own OpenAI-compatible Serverless Endpoint on RunPod with multiple embedding models and fast inference for RAG and more! @@ -81,7 +81,7 @@ Below is a summary of the available RunPod Worker images, categorized by image s | CUDA Version | Stable Image Tag | Development Image Tag | Note | |--------------|-----------------------------------|-----------------------------------|----------------------------------------------------------------------| -| 12.1.0 | `runpod/worker-v1-vllm:v2.4.0stable-cuda12.1.0` | `runpod/worker-v1-vllm:v2.4.0dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. | +| 12.1.0 | `runpod/worker-v1-vllm:v2.5.0stable-cuda12.1.0` | `runpod/worker-v1-vllm:v2.5.0dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. | diff --git a/worker-config.json b/worker-config.json index 27173e8..10f0e53 100644 --- a/worker-config.json +++ b/worker-config.json @@ -1,5 +1,66 @@ { "versions": { + "0.8.5": { + "imageName": "runpod/worker-v1-vllm:v2.5.0stable-cuda12.1.0", + "minimumCudaVersion": "12.1", + "categories": [ + { + "title": "LLM Settings", + "settings": [ + "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", + "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", + "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", + "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", + "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", + "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS", + "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", + "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", + "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", + "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", + "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", + "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", + "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", + "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", + "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", + "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", + "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", + "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", + "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" + ] + }, + { + "title": "Tokenizer Settings", + "settings": [ + "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" + ] + }, + { + "title": "System Settings", + "settings": [ + "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", + "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" + ] + }, + { + "title": "Streaming Settings", + "settings": [ + "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" + ] + }, + { + "title": "OpenAI Settings", + "settings": [ + "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" + ] + }, + { + "title": "Serverless Settings", + "settings": [ + "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" + ] + } + ] + }, "0.8.4": { "imageName": "runpod/worker-v1-vllm:v2.4.0stable-cuda12.1.0", "minimumCudaVersion": "12.1",