From 3dad3a754cca30cd241ecee150ad22c170790c32 Mon Sep 17 00:00:00 2001 From: pandyamarut Date: Fri, 6 Sep 2024 12:21:34 -0700 Subject: [PATCH 1/2] update vllm Signed-off-by: pandyamarut --- README.md | 6 ++--- docker-bake.hcl | 2 +- worker-config.json | 60 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6fc69d7..3e07341 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https: ### 1. UI for Deploying vLLM Worker on RunPod console: ![Demo of Deploying vLLM Worker on RunPod console with new UI](media/ui_demo.gif) -### 2. Worker vLLM `v1.3.0` with vLLM `0.5.5` now available under `stable` tags +### 2. Worker vLLM `v1.3.1` with vLLM `0.5.5` now available under `stable` tags -Update v1.3.0 is now available, use the image tag `runpod/worker-v1-vllm:v1.3.0stable-cuda12.1.0`. +Update v1.3.1 is now available, use the image tag `runpod/worker-v1-vllm:v1.3.1stable-cuda12.1.0`. ### 3. OpenAI-Compatible [Embedding Worker](https://github.com/runpod-workers/worker-infinity-embedding) Released Deploy your own OpenAI-compatible Serverless Endpoint on RunPod with multiple embedding models and fast inference for RAG and more! @@ -82,7 +82,7 @@ Below is a summary of the available RunPod Worker images, categorized by image s | CUDA Version | Stable Image Tag | Development Image Tag | Note | |--------------|-----------------------------------|-----------------------------------|----------------------------------------------------------------------| -| 12.1.0 | `runpod/worker-v1-vllm:stable-cuda12.1.0` | `runpod/worker-v1-vllm:dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. | +| 12.1.0 | `runpod/worker-v1-vllm:v1.3.1stable-cuda12.1.0` | `runpod/worker-v1-vllm:v1.3.1dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. | diff --git a/docker-bake.hcl b/docker-bake.hcl index 4b6939f..7d825a3 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -7,7 +7,7 @@ variable "REPOSITORY" { } variable "BASE_IMAGE_VERSION" { - default = "stable" + default = "v1.2.0preview" } group "all" { diff --git a/worker-config.json b/worker-config.json index bb25d01..391fb62 100644 --- a/worker-config.json +++ b/worker-config.json @@ -1,5 +1,65 @@ { "versions": { + "0.5.5": { + "imageName": "runpod/worker-v1-vllm:v1.3.1stable-cuda12.1.0", + "minimumCudaVersion": "12.1", + "categories": [ + { + "title": "LLM Settings", + "settings": [ + "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", + "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", + "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", + "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", + "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", + "DISABLE_SLIDING_WINDOW", "USE_V2_BLOCK_MANAGER", "NUM_LOOKAHEAD_SLOTS", + "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", + "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", + "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", + "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", + "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", + "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", + "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", + "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", + "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", + "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", + "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", + "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST" + ] + }, + { + "title": "Tokenizer Settings", + "settings": [ + "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" + ] + }, + { + "title": "System Settings", + "settings": [ + "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", + "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" + ] + }, + { + "title": "Streaming Settings", + "settings": [ + "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" + ] + }, + { + "title": "OpenAI Settings", + "settings": [ + "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" + ] + }, + { + "title": "Serverless Settings", + "settings": [ + "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" + ] + } + ] + }, "0.5.4": { "imageName": "runpod/worker-v1-vllm:v1.2.0stable-cuda12.1.0", "minimumCudaVersion": "12.1", From 0d794a39143479b12f4fe0f97cb165dbcb73b454 Mon Sep 17 00:00:00 2001 From: pandyamarut Date: Fri, 6 Sep 2024 12:26:09 -0700 Subject: [PATCH 2/2] fix Signed-off-by: pandyamarut --- docker-bake.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 7d825a3..4b6939f 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -7,7 +7,7 @@ variable "REPOSITORY" { } variable "BASE_IMAGE_VERSION" { - default = "v1.2.0preview" + default = "stable" } group "all" {