diff --git a/.runpod/hub.json b/.runpod/hub.json index 04578ba..04c810d 100644 --- a/.runpod/hub.json +++ b/.runpod/hub.json @@ -935,7 +935,7 @@ "name": "Max Concurrency", "type": "number", "description": "Max concurrent requests per worker. vLLM has an internal queue, so you don't have to worry about limiting by VRAM, this is for improving scaling/load balancing efficiency", - "default": 300, + "default": 30, "advanced": true } }, diff --git a/README.md b/README.md index 623ca0b..8cd7c2f 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Configure worker-vllm using environment variables: | `ENABLE_AUTO_TOOL_CHOICE` | Enable automatic tool selection | false | boolean (true or false) | | `TOOL_CALL_PARSER` | Parser for tool calls | | "mistral", "hermes", "llama3_json", "granite", "deepseek_v3", etc. | | `OPENAI_SERVED_MODEL_NAME_OVERRIDE` | Override served model name in API | | String | -| `MAX_CONCURRENCY` | Maximum concurrent requests | 300 | Integer | +| `MAX_CONCURRENCY` | Maximum concurrent requests | 30 | Integer | For the complete list of all available environment variables, examples, and detailed descriptions: **[Configuration](docs/configuration.md)** diff --git a/docs/configuration.md b/docs/configuration.md index f886064..ccc6e48 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -119,7 +119,7 @@ The way this works is that the first request will have a batch size of `DEFAULT_ | Variable | Default | Type/Choices | Description | | ---------------------- | ------- | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `MAX_CONCURRENCY` | `300` | `int` | Max concurrent requests per worker. vLLM has an internal queue, so you don't have to worry about limiting by VRAM, this is for improving scaling/load balancing efficiency | +| `MAX_CONCURRENCY` | `30` | `int` | Max concurrent requests per worker. vLLM has an internal queue, so you don't have to worry about limiting by VRAM, this is for improving scaling/load balancing efficiency | | `DISABLE_LOG_STATS` | False | `bool` | Enables or disables vLLM stats logging. | | `DISABLE_LOG_REQUESTS` | False | `bool` | Enables or disables vLLM request logging. | diff --git a/docs/conventions.md b/docs/conventions.md index 8531ece..31b76da 100644 --- a/docs/conventions.md +++ b/docs/conventions.md @@ -51,7 +51,8 @@ RunPod Request → handler.py → JobInput → Engine Selection → vLLM Generat - `src/engine_args.py`: Centralized configuration management - `src/constants.py`: Default values for core settings -- `worker-config.json`: UI form generation for RunPod console +- `.runpod/hub.json`: Hub UI configuration (CRITICAL: always update when changing defaults) +- `worker-config.json`: UI form generation for RunPod console (if exists) ## Core Development Concepts @@ -222,7 +223,7 @@ src/ ### 2. **Concurrency Patterns** -- **Max Concurrency**: 300 concurrent requests by default +- **Max Concurrency**: 30 concurrent requests by default - **vLLM Queuing**: Internal request batching and scheduling - **RunPod Integration**: Concurrency modifier for auto-scaling diff --git a/src/constants.py b/src/constants.py index a75b5f1..b312e9e 100644 --- a/src/constants.py +++ b/src/constants.py @@ -1,4 +1,4 @@ DEFAULT_BATCH_SIZE = 50 -DEFAULT_MAX_CONCURRENCY = 300 +DEFAULT_MAX_CONCURRENCY = 30 DEFAULT_BATCH_SIZE_GROWTH_FACTOR = 3 DEFAULT_MIN_BATCH_SIZE = 1 \ No newline at end of file diff --git a/worker-config.json b/worker-config.json deleted file mode 100644 index 5453c71..0000000 --- a/worker-config.json +++ /dev/null @@ -1,1514 +0,0 @@ -{ - "versions": { - "0.10.0": { - "imageName": "runpod/worker-v1-vllm:v2.8.0stable-cuda12.1.0", - "minimumCudaVersion": "12.1", - "categories": [ - { - "title": "LLM Settings", - "settings": [ - "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", - "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", - "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", - "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", - "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", - "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS", - "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", - "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", - "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", - "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", - "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", - "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", - "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", - "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", - "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", - "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", - "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" - ] - }, - { - "title": "Tokenizer Settings", - "settings": [ - "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" - ] - }, - { - "title": "System Settings", - "settings": [ - "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", - "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" - ] - }, - { - "title": "Streaming Settings", - "settings": [ - "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" - ] - }, - { - "title": "OpenAI Settings", - "settings": [ - "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" - ] - }, - { - "title": "Serverless Settings", - "settings": [ - "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" - ] - } - ] - }, - "0.9.0": { - "imageName": "runpod/worker-v1-vllm:v2.6.0stable-cuda12.1.0", - "minimumCudaVersion": "12.1", - "categories": [ - { - "title": "LLM Settings", - "settings": [ - "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", - "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", - "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", - "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", - "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", - "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS", - "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", - "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", - "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", - "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", - "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", - "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", - "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", - "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", - "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", - "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", - "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" - ] - }, - { - "title": "Tokenizer Settings", - "settings": [ - "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" - ] - }, - { - "title": "System Settings", - "settings": [ - "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", - "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" - ] - }, - { - "title": "Streaming Settings", - "settings": [ - "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" - ] - }, - { - "title": "OpenAI Settings", - "settings": [ - "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" - ] - }, - { - "title": "Serverless Settings", - "settings": [ - "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" - ] - } - ] - }, - "0.9.1": { - "imageName": "runpod/worker-v1-vllm:v2.7.0stable-cuda12.1.0", - "minimumCudaVersion": "12.1", - "categories": [ - { - "title": "LLM Settings", - "settings": [ - "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", - "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", - "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", - "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", - "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", - "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS", - "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", - "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", - "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", - "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", - "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", - "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", - "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", - "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", - "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", - "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", - "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" - ] - }, - { - "title": "Tokenizer Settings", - "settings": [ - "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" - ] - }, - { - "title": "System Settings", - "settings": [ - "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", - "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" - ] - }, - { - "title": "Streaming Settings", - "settings": [ - "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" - ] - }, - { - "title": "OpenAI Settings", - "settings": [ - "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" - ] - }, - { - "title": "Serverless Settings", - "settings": [ - "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" - ] - } - ] - }, - "0.9.0": { - "imageName": "runpod/worker-v1-vllm:v2.6.0stable-cuda12.1.0", - "minimumCudaVersion": "12.1", - "categories": [ - { - "title": "LLM Settings", - "settings": [ - "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", - "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", - "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", - "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", - "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", - "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS", - "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", - "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", - "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", - "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", - "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", - "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", - "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", - "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", - "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", - "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", - "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" - ] - }, - { - "title": "Tokenizer Settings", - "settings": [ - "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" - ] - }, - { - "title": "System Settings", - "settings": [ - "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", - "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" - ] - }, - { - "title": "Streaming Settings", - "settings": [ - "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" - ] - }, - { - "title": "OpenAI Settings", - "settings": [ - "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" - ] - }, - { - "title": "Serverless Settings", - "settings": [ - "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" - ] - } - ] - }, - "0.8.5": { - "imageName": "runpod/worker-v1-vllm:v2.5.0stable-cuda12.1.0", - "minimumCudaVersion": "12.1", - "categories": [ - { - "title": "LLM Settings", - "settings": [ - "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", - "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", - "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", - "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", - "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", - "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS", - "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", - "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", - "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", - "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", - "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", - "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", - "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", - "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", - "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", - "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", - "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" - ] - }, - { - "title": "Tokenizer Settings", - "settings": [ - "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" - ] - }, - { - "title": "System Settings", - "settings": [ - "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", - "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" - ] - }, - { - "title": "Streaming Settings", - "settings": [ - "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" - ] - }, - { - "title": "OpenAI Settings", - "settings": [ - "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" - ] - }, - { - "title": "Serverless Settings", - "settings": [ - "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" - ] - } - ] - }, - "0.8.4": { - "imageName": "runpod/worker-v1-vllm:v2.4.0stable-cuda12.1.0", - "minimumCudaVersion": "12.1", - "categories": [ - { - "title": "LLM Settings", - "settings": [ - "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", - "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", - "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", - "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", - "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", - "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS", - "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", - "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", - "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", - "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", - "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", - "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", - "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", - "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", - "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", - "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", - "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" - ] - }, - { - "title": "Tokenizer Settings", - "settings": [ - "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" - ] - }, - { - "title": "System Settings", - "settings": [ - "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", - "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" - ] - }, - { - "title": "Streaming Settings", - "settings": [ - "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" - ] - }, - { - "title": "OpenAI Settings", - "settings": [ - "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" - ] - }, - { - "title": "Serverless Settings", - "settings": [ - "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" - ] - } - ] - }, - "0.8.3": { - "imageName": "runpod/worker-v1-vllm:v2.3.0stable-cuda12.1.0", - "minimumCudaVersion": "12.1", - "categories": [ - { - "title": "LLM Settings", - "settings": [ - "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", - "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", - "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", - "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", - "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", - "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS", - "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", - "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", - "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", - "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", - "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", - "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", - "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", - "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", - "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", - "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", - "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" - ] - }, - { - "title": "Tokenizer Settings", - "settings": [ - "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" - ] - }, - { - "title": "System Settings", - "settings": [ - "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", - "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" - ] - }, - { - "title": "Streaming Settings", - "settings": [ - "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" - ] - }, - { - "title": "OpenAI Settings", - "settings": [ - "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" - ] - }, - { - "title": "Serverless Settings", - "settings": [ - "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" - ] - } - ] - }, - "0.8.2": { - "imageName": "runpod/worker-v1-vllm:v2.2.0stable-cuda12.1.0", - "minimumCudaVersion": "12.1", - "categories": [ - { - "title": "LLM Settings", - "settings": [ - "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", - "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", - "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", - "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", - "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", - "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS", - "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", - "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", - "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", - "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", - "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", - "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", - "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", - "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", - "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", - "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", - "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" - ] - }, - { - "title": "Tokenizer Settings", - "settings": [ - "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" - ] - }, - { - "title": "System Settings", - "settings": [ - "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", - "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" - ] - }, - { - "title": "Streaming Settings", - "settings": [ - "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" - ] - }, - { - "title": "OpenAI Settings", - "settings": [ - "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" - ] - }, - { - "title": "Serverless Settings", - "settings": [ - "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" - ] - } - ] - }, - "0.7.3": { - "imageName": "runpod/worker-v1-vllm:v2.1.0stable-cuda12.1.0", - "minimumCudaVersion": "12.1", - "categories": [ - { - "title": "LLM Settings", - "settings": [ - "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", - "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", - "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", - "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", - "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", - "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS", - "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", - "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", - "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", - "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", - "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", - "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", - "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", - "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", - "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", - "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", - "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" - ] - }, - { - "title": "Tokenizer Settings", - "settings": [ - "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" - ] - }, - { - "title": "System Settings", - "settings": [ - "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", - "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" - ] - }, - { - "title": "Streaming Settings", - "settings": [ - "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" - ] - }, - { - "title": "OpenAI Settings", - "settings": [ - "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" - ] - }, - { - "title": "Serverless Settings", - "settings": [ - "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" - ] - } - ] - }, - "0.6.6": { - "imageName": "runpod/worker-v1-vllm:v1.8.0stable-cuda12.1.0", - "minimumCudaVersion": "12.1", - "categories": [ - { - "title": "LLM Settings", - "settings": [ - "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", - "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", - "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", - "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", - "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", - "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS", - "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", - "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", - "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", - "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", - "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", - "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", - "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", - "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", - "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", - "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", - "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" - ] - }, - { - "title": "Tokenizer Settings", - "settings": [ - "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" - ] - }, - { - "title": "System Settings", - "settings": [ - "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", - "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" - ] - }, - { - "title": "Streaming Settings", - "settings": [ - "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" - ] - }, - { - "title": "OpenAI Settings", - "settings": [ - "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" - ] - }, - { - "title": "Serverless Settings", - "settings": [ - "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" - ] - } - ] - }, - "0.7.0": { - "imageName": "runpod/worker-v1-vllm:v1.9.0stable-cuda12.1.0", - "minimumCudaVersion": "12.1", - "categories": [ - { - "title": "LLM Settings", - "settings": [ - "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", - "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", - "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", - "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", - "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", - "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS", - "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", - "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", - "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", - "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", - "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", - "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", - "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", - "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", - "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", - "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", - "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" - ] - }, - { - "title": "Tokenizer Settings", - "settings": [ - "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" - ] - }, - { - "title": "System Settings", - "settings": [ - "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", - "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" - ] - }, - { - "title": "Streaming Settings", - "settings": [ - "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" - ] - }, - { - "title": "OpenAI Settings", - "settings": [ - "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" - ] - }, - { - "title": "Serverless Settings", - "settings": [ - "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" - ] - } - ] - }, - "0.6.4": { - "imageName": "runpod/worker-v1-vllm:v1.7.0stable-cuda12.1.0", - "minimumCudaVersion": "12.1", - "categories": [ - { - "title": "LLM Settings", - "settings": [ - "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", - "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", - "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", - "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", - "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", - "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS", - "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", - "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", - "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", - "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", - "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", - "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", - "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", - "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", - "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", - "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", - "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" - ] - }, - { - "title": "Tokenizer Settings", - "settings": [ - "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" - ] - }, - { - "title": "System Settings", - "settings": [ - "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", - "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" - ] - }, - { - "title": "Streaming Settings", - "settings": [ - "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" - ] - }, - { - "title": "OpenAI Settings", - "settings": [ - "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" - ] - }, - { - "title": "Serverless Settings", - "settings": [ - "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" - ] - } - ] - }, - "0.6.3": { - "imageName": "runpod/worker-v1-vllm:v1.6.0stable-cuda12.1.0", - "minimumCudaVersion": "12.1", - "categories": [ - { - "title": "LLM Settings", - "settings": [ - "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", - "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", - "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", - "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", - "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", - "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS", - "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", - "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", - "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", - "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", - "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", - "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", - "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", - "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", - "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", - "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", - "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" - ] - }, - { - "title": "Tokenizer Settings", - "settings": [ - "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" - ] - }, - { - "title": "System Settings", - "settings": [ - "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", - "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" - ] - }, - { - "title": "Streaming Settings", - "settings": [ - "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" - ] - }, - { - "title": "OpenAI Settings", - "settings": [ - "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" - ] - }, - { - "title": "Serverless Settings", - "settings": [ - "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" - ] - } - ] - } - }, - "schema": { - "TOKENIZER": { - "env_var_name": "TOKENIZER", - "value": "", - "title": "Tokenizer", - "description": "Name or path of the Hugging Face tokenizer to use.", - "required": false, - "type": "text" - }, - "TOKENIZER_MODE": { - "env_var_name": "TOKENIZER_MODE", - "value": "auto", - "title": "Tokenizer Mode", - "description": "The tokenizer mode.", - "required": false, - "type": "select", - "options": [ - { "value": "auto", "label": "auto" }, - { "value": "slow", "label": "slow" } - ] - }, - "SKIP_TOKENIZER_INIT": { - "env_var_name": "SKIP_TOKENIZER_INIT", - "value": false, - "title": "Skip Tokenizer Init", - "description": "Skip initialization of tokenizer and detokenizer.", - "required": false, - "type": "toggle" - }, - "TRUST_REMOTE_CODE": { - "env_var_name": "TRUST_REMOTE_CODE", - "value": false, - "title": "Trust Remote Code", - "description": "Trust remote code from Hugging Face.", - "required": false, - "type": "toggle" - }, - "DOWNLOAD_DIR": { - "env_var_name": "DOWNLOAD_DIR", - "value": "", - "title": "Download Directory", - "description": "Directory to download and load the weights.", - "required": false, - "type": "text" - }, - "LOAD_FORMAT": { - "env_var_name": "LOAD_FORMAT", - "value": "auto", - "title": "Load Format", - "description": "The format of the model weights to load.", - "required": false, - "type": "select", - "options": [ - { "value": "auto", "label": "auto" }, - { "value": "pt", "label": "pt" }, - { "value": "safetensors", "label": "safetensors" }, - { "value": "npcache", "label": "npcache" }, - { "value": "dummy", "label": "dummy" }, - { "value": "tensorizer", "label": "tensorizer" }, - { "value": "bitsandbytes", "label": "bitsandbytes" } - ] - }, - "DTYPE": { - "env_var_name": "DTYPE", - "value": "auto", - "title": "Data Type", - "description": "Data type for model weights and activations.", - "required": false, - "type": "select", - "options": [ - { "value": "auto", "label": "auto" }, - { "value": "half", "label": "half" }, - { "value": "float16", "label": "float16" }, - { "value": "bfloat16", "label": "bfloat16" }, - { "value": "float", "label": "float" }, - { "value": "float32", "label": "float32" } - ] - }, - "KV_CACHE_DTYPE": { - "env_var_name": "KV_CACHE_DTYPE", - "value": "auto", - "title": "KV Cache Data Type", - "description": "Data type for KV cache storage.", - "required": false, - "type": "select", - "options": [ - { "value": "auto", "label": "auto" }, - { "value": "fp8", "label": "fp8" } - ] - }, - "QUANTIZATION_PARAM_PATH": { - "env_var_name": "QUANTIZATION_PARAM_PATH", - "value": "", - "title": "Quantization Param Path", - "description": "Path to the JSON file containing the KV cache scaling factors.", - "required": false, - "type": "text" - }, - "MAX_MODEL_LEN": { - "env_var_name": "MAX_MODEL_LEN", - "value": "", - "title": "Max Model Length", - "description": "Model context length.", - "required": false, - "type": "number" - }, - "GUIDED_DECODING_BACKEND": { - "env_var_name": "GUIDED_DECODING_BACKEND", - "value": "outlines", - "title": "Guided Decoding Backend", - "description": "Which engine will be used for guided decoding by default.", - "required": false, - "type": "select", - "options": [ - { "value": "outlines", "label": "outlines" }, - { "value": "lm-format-enforcer", "label": "lm-format-enforcer" } - ] - }, - "DISTRIBUTED_EXECUTOR_BACKEND": { - "env_var_name": "DISTRIBUTED_EXECUTOR_BACKEND", - "value": "", - "title": "Distributed Executor Backend", - "description": "Backend to use for distributed serving.", - "required": false, - "type": "select", - "options": [ - { "value": "ray", "label": "ray" }, - { "value": "mp", "label": "mp" } - ] - }, - "WORKER_USE_RAY": { - "env_var_name": "WORKER_USE_RAY", - "value": false, - "title": "Worker Use Ray", - "description": "Deprecated, use --distributed-executor-backend=ray.", - "required": false, - "type": "toggle" - }, - "RAY_WORKERS_USE_NSIGHT": { - "env_var_name": "RAY_WORKERS_USE_NSIGHT", - "value": false, - "title": "Ray Workers Use Nsight", - "description": "If specified, use nsight to profile Ray workers.", - "required": false, - "type": "toggle" - }, - "PIPELINE_PARALLEL_SIZE": { - "env_var_name": "PIPELINE_PARALLEL_SIZE", - "value": 1, - "title": "Pipeline Parallel Size", - "description": "Number of pipeline stages.", - "required": false, - "type": "number" - }, - "TENSOR_PARALLEL_SIZE": { - "env_var_name": "TENSOR_PARALLEL_SIZE", - "value": 1, - "title": "Tensor Parallel Size", - "description": "Number of tensor parallel replicas.", - "required": false, - "type": "number" - }, - "MAX_PARALLEL_LOADING_WORKERS": { - "env_var_name": "MAX_PARALLEL_LOADING_WORKERS", - "value": "", - "title": "Max Parallel Loading Workers", - "description": "Load model sequentially in multiple batches.", - "required": false, - "type": "number" - }, - "ENABLE_PREFIX_CACHING": { - "env_var_name": "ENABLE_PREFIX_CACHING", - "value": false, - "title": "Enable Prefix Caching", - "description": "Enables automatic prefix caching.", - "required": false, - "type": "toggle" - }, - "DISABLE_SLIDING_WINDOW": { - "env_var_name": "DISABLE_SLIDING_WINDOW", - "value": false, - "title": "Disable Sliding Window", - "description": "Disables sliding window, capping to sliding window size.", - "required": false, - "type": "toggle" - }, - "USE_V2_BLOCK_MANAGER": { - "env_var_name": "USE_V2_BLOCK_MANAGER", - "value": false, - "title": "Use V2 Block Manager", - "description": "Use BlockSpaceMangerV2.", - "required": false, - "type": "toggle" - }, - "NUM_LOOKAHEAD_SLOTS": { - "env_var_name": "NUM_LOOKAHEAD_SLOTS", - "value": 0, - "title": "Num Lookahead Slots", - "description": "Experimental scheduling config necessary for speculative decoding.", - "required": false, - "type": "number" - }, - "SEED": { - "env_var_name": "SEED", - "value": 0, - "title": "Seed", - "description": "Random seed for operations.", - "required": false, - "type": "number" - }, - "NUM_GPU_BLOCKS_OVERRIDE": { - "env_var_name": "NUM_GPU_BLOCKS_OVERRIDE", - "value": "", - "title": "Num GPU Blocks Override", - "description": "If specified, ignore GPU profiling result and use this number of GPU blocks.", - "required": false, - "type": "number" - }, - "MAX_NUM_BATCHED_TOKENS": { - "env_var_name": "MAX_NUM_BATCHED_TOKENS", - "value": "", - "title": "Max Num Batched Tokens", - "description": "Maximum number of batched tokens per iteration.", - "required": false, - "type": "number" - }, - "MAX_NUM_SEQS": { - "env_var_name": "MAX_NUM_SEQS", - "value": 256, - "title": "Max Num Seqs", - "description": "Maximum number of sequences per iteration.", - "required": false, - "type": "number" - }, - "MAX_LOGPROBS": { - "env_var_name": "MAX_LOGPROBS", - "value": 20, - "title": "Max Logprobs", - "description": "Max number of log probs to return when logprobs is specified in SamplingParams.", - "required": false, - "type": "number" - }, - "DISABLE_LOG_STATS": { - "env_var_name": "DISABLE_LOG_STATS", - "value": false, - "title": "Disable Log Stats", - "description": "Disable logging statistics.", - "required": false, - "type": "toggle" - }, - "QUANTIZATION": { - "env_var_name": "QUANTIZATION", - "value": "", - "title": "Quantization", - "description": "Method used to quantize the weights.\nif the `Load Format` is 'bitsandbytes' then `Quantization` will be forced to 'bitsandbytes'", - "required": false, - "type": "select", - "options": [ - { "value": "None", "label": "None" }, - { "value": "awq", "label": "AWQ" }, - { "value": "squeezellm", "label": "SqueezeLLM" }, - { "value": "gptq", "label": "GPTQ" }, - { "value": "bitsandbytes", "label": "bitsandbytes" } - ] - }, - "ROPE_SCALING": { - "env_var_name": "ROPE_SCALING", - "value": "", - "title": "RoPE Scaling", - "description": "RoPE scaling configuration in JSON format.", - "required": false, - "type": "text" - }, - "ROPE_THETA": { - "env_var_name": "ROPE_THETA", - "value": "", - "title": "RoPE Theta", - "description": "RoPE theta. Use with rope_scaling.", - "required": false, - "type": "number" - }, - "TOKENIZER_POOL_SIZE": { - "env_var_name": "TOKENIZER_POOL_SIZE", - "value": 0, - "title": "Tokenizer Pool Size", - "description": "Size of tokenizer pool to use for asynchronous tokenization.", - "required": false, - "type": "number" - }, - "TOKENIZER_POOL_TYPE": { - "env_var_name": "TOKENIZER_POOL_TYPE", - "value": "ray", - "title": "Tokenizer Pool Type", - "description": "Type of tokenizer pool to use for asynchronous tokenization.", - "required": false, - "type": "text" - }, - "TOKENIZER_POOL_EXTRA_CONFIG": { - "env_var_name": "TOKENIZER_POOL_EXTRA_CONFIG", - "value": "", - "title": "Tokenizer Pool Extra Config", - "description": "Extra config for tokenizer pool.", - "required": false, - "type": "text" - }, - "ENABLE_LORA": { - "env_var_name": "ENABLE_LORA", - "value": false, - "title": "Enable LoRA", - "description": "If True, enable handling of LoRA adapters.", - "required": false, - "type": "toggle" - }, - "MAX_LORAS": { - "env_var_name": "MAX_LORAS", - "value": 1, - "title": "Max LoRAs", - "description": "Max number of LoRAs in a single batch.", - "required": false, - "type": "number" - }, - "MAX_LORA_RANK": { - "env_var_name": "MAX_LORA_RANK", - "value": 16, - "title": "Max LoRA Rank", - "description": "Max LoRA rank.", - "required": false, - "type": "number" - }, - "LORA_EXTRA_VOCAB_SIZE": { - "env_var_name": "LORA_EXTRA_VOCAB_SIZE", - "value": 256, - "title": "LoRA Extra Vocab Size", - "description": "Maximum size of extra vocabulary for LoRA adapters.", - "required": false, - "type": "number" - }, - "LORA_DTYPE": { - "env_var_name": "LORA_DTYPE", - "value": "auto", - "title": "LoRA Data Type", - "description": "Data type for LoRA.", - "required": false, - "type": "select", - "options": [ - { "value": "auto", "label": "auto" }, - { "value": "float16", "label": "float16" }, - { "value": "bfloat16", "label": "bfloat16" }, - { "value": "float32", "label": "float32" } - ] - }, - "LONG_LORA_SCALING_FACTORS": { - "env_var_name": "LONG_LORA_SCALING_FACTORS", - "value": "", - "title": "Long LoRA Scaling Factors", - "description": "Specify multiple scaling factors for LoRA adapters.", - "required": false, - "type": "text" - }, - "MAX_CPU_LORAS": { - "env_var_name": "MAX_CPU_LORAS", - "value": "", - "title": "Max CPU LoRAs", - "description": "Maximum number of LoRAs to store in CPU memory.", - "required": false, - "type": "number" - }, - "FULLY_SHARDED_LORAS": { - "env_var_name": "FULLY_SHARDED_LORAS", - "value": false, - "title": "Fully Sharded LoRAs", - "description": "Enable fully sharded LoRA layers.", - "required": false, - "type": "toggle" - }, - "DEVICE": { - "env_var_name": "DEVICE", - "value": "auto", - "title": "Device", - "description": "Device type for vLLM execution.", - "required": false, - "type": "select", - "options": [ - { "value": "auto", "label": "auto" }, - { "value": "cuda", "label": "cuda" }, - { "value": "neuron", "label": "neuron" }, - { "value": "cpu", "label": "cpu" }, - { "value": "openvino", "label": "openvino" }, - { "value": "tpu", "label": "tpu" }, - { "value": "xpu", "label": "xpu" } - ] - }, - "SCHEDULER_DELAY_FACTOR": { - "env_var_name": "SCHEDULER_DELAY_FACTOR", - "value": 0.0, - "title": "Scheduler Delay Factor", - "description": "Apply a delay before scheduling next prompt.", - "required": false, - "type": "number" - }, - "ENABLE_CHUNKED_PREFILL": { - "env_var_name": "ENABLE_CHUNKED_PREFILL", - "value": false, - "title": "Enable Chunked Prefill", - "description": "Enable chunked prefill requests.", - "required": false, - "type": "toggle" - }, - "SPECULATIVE_MODEL": { - "env_var_name": "SPECULATIVE_MODEL", - "value": "", - "title": "Speculative Model", - "description": "The name of the draft model to be used in speculative decoding.", - "required": false, - "type": "text" - }, - "NUM_SPECULATIVE_TOKENS": { - "env_var_name": "NUM_SPECULATIVE_TOKENS", - "value": "", - "title": "Num Speculative Tokens", - "description": "The number of speculative tokens to sample from the draft model.", - "required": false, - "type": "number" - }, - "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE": { - "env_var_name": "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", - "value": "", - "title": "Speculative Draft Tensor Parallel Size", - "description": "Number of tensor parallel replicas for the draft model.", - "required": false, - "type": "number" - }, - "SPECULATIVE_MAX_MODEL_LEN": { - "env_var_name": "SPECULATIVE_MAX_MODEL_LEN", - "value": "", - "title": "Speculative Max Model Length", - "description": "The maximum sequence length supported by the draft model.", - "required": false, - "type": "number" - }, - "SPECULATIVE_DISABLE_BY_BATCH_SIZE": { - "env_var_name": "SPECULATIVE_DISABLE_BY_BATCH_SIZE", - "value": "", - "title": "Speculative Disable by Batch Size", - "description": "Disable speculative decoding if the number of enqueue requests is larger than this value.", - "required": false, - "type": "number" - }, - "NGRAM_PROMPT_LOOKUP_MAX": { - "env_var_name": "NGRAM_PROMPT_LOOKUP_MAX", - "value": "", - "title": "Ngram Prompt Lookup Max", - "description": "Max size of window for ngram prompt lookup in speculative decoding.", - "required": false, - "type": "number" - }, - "NGRAM_PROMPT_LOOKUP_MIN": { - "env_var_name": "NGRAM_PROMPT_LOOKUP_MIN", - "value": "", - "title": "Ngram Prompt Lookup Min", - "description": "Min size of window for ngram prompt lookup in speculative decoding.", - "required": false, - "type": "number" - }, - "SPEC_DECODING_ACCEPTANCE_METHOD": { - "env_var_name": "SPEC_DECODING_ACCEPTANCE_METHOD", - "value": "rejection_sampler", - "title": "Speculative Decoding Acceptance Method", - "description": "Specify the acceptance method for draft token verification in speculative decoding.", - "required": false, - "type": "select", - "options": [ - { "value": "rejection_sampler", "label": "rejection_sampler" }, - { "value": "typical_acceptance_sampler", "label": "typical_acceptance_sampler" } - ] - }, - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD": { - "env_var_name": "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", - "value": "", - "title": "Typical Acceptance Sampler Posterior Threshold", - "description": "Set the lower bound threshold for the posterior probability of a token to be accepted.", - "required": false, - "type": "number" - }, - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA": { - "env_var_name": "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", - "value": "", - "title": "Typical Acceptance Sampler Posterior Alpha", - "description": "A scaling factor for the entropy-based threshold for token acceptance.", - "required": false, - "type": "number" - }, - "MODEL_LOADER_EXTRA_CONFIG": { - "env_var_name": "MODEL_LOADER_EXTRA_CONFIG", - "value": "", - "title": "Model Loader Extra Config", - "description": "Extra config for model loader.", - "required": false, - "type": "text" - }, - "PREEMPTION_MODE": { - "env_var_name": "PREEMPTION_MODE", - "value": "", - "title": "Preemption Mode", - "description": "If 'recompute', the engine performs preemption-aware recomputation. If 'save', the engine saves activations into the CPU memory as preemption happens.", - "required": false, - "type": "text" - }, - "PREEMPTION_CHECK_PERIOD": { - "env_var_name": "PREEMPTION_CHECK_PERIOD", - "value": 1.0, - "title": "Preemption Check Period", - "description": "How frequently the engine checks if a preemption happens.", - "required": false, - "type": "number" - }, - "PREEMPTION_CPU_CAPACITY": { - "env_var_name": "PREEMPTION_CPU_CAPACITY", - "value": 2, - "title": "Preemption CPU Capacity", - "description": "The percentage of CPU memory used for the saved activations.", - "required": false, - "type": "number" - }, - "MAX_LOG_LEN": { - "env_var_name": "MAX_LOG_LEN", - "value": "", - "title": "Max Log Length", - "description": "Max number of characters or ID numbers being printed in log.", - "required": false, - "type": "number" - }, - "DISABLE_LOGGING_REQUEST": { - "env_var_name": "DISABLE_LOGGING_REQUEST", - "value": false, - "title": "Disable Logging Request", - "description": "Disable logging requests.", - "required": false, - "type": "toggle" - }, - "TOKENIZER_NAME": { - "env_var_name": "TOKENIZER_NAME", - "value": "", - "title": "Tokenizer Name", - "description": "Tokenizer repo to use a different tokenizer than the model's default", - "required": false, - "type": "text" - }, - "TOKENIZER_REVISION": { - "env_var_name": "TOKENIZER_REVISION", - "value": "", - "title": "Tokenizer Revision", - "description": "Tokenizer revision to load", - "required": false, - "type": "text" - }, - "CUSTOM_CHAT_TEMPLATE": { - "env_var_name": "CUSTOM_CHAT_TEMPLATE", - "value": "", - "title": "Custom Chat Template", - "description": "Custom chat jinja template", - "required": false, - "type": "text" - }, - "GPU_MEMORY_UTILIZATION": { - "env_var_name": "GPU_MEMORY_UTILIZATION", - "value": "0.95", - "title": "GPU Memory Utilization", - "description": "Sets GPU VRAM utilization", - "required": false, - "type": "number" - }, - "BLOCK_SIZE": { - "env_var_name": "BLOCK_SIZE", - "value": "16", - "title": "Block Size", - "description": "Token block size for contiguous chunks of tokens", - "required": false, - "type": "number" - }, - "SWAP_SPACE": { - "env_var_name": "SWAP_SPACE", - "value": "4", - "title": "Swap Space", - "description": "CPU swap space size (GiB) per GPU", - "required": false, - "type": "number" - }, - "ENFORCE_EAGER": { - "env_var_name": "ENFORCE_EAGER", - "value": false, - "title": "Enforce Eager", - "description": "Always use eager-mode PyTorch. If False (0), will use eager mode and CUDA graph in hybrid for maximal performance and flexibility", - "required": false, - "type": "toggle" - }, - "MAX_SEQ_LEN_TO_CAPTURE": { - "env_var_name": "MAX_SEQ_LEN_TO_CAPTURE", - "value": "8192", - "title": "CUDA Graph Max Content Length", - "description": "Maximum context length covered by CUDA graphs. If a sequence has context length larger than this, we fall back to eager mode", - "required": false, - "type": "number" - }, - "DISABLE_CUSTOM_ALL_REDUCE": { - "env_var_name": "DISABLE_CUSTOM_ALL_REDUCE", - "value": false, - "title": "Disable Custom All Reduce", - "description": "Enables or disables custom all reduce", - "required": false, - "type": "toggle" - }, - "DEFAULT_BATCH_SIZE": { - "env_var_name": "DEFAULT_BATCH_SIZE", - "value": "50", - "title": "Default Final Batch Size", - "description": "Default and Maximum batch size for token streaming to reduce HTTP calls", - "required": false, - "type": "number" - }, - "DEFAULT_MIN_BATCH_SIZE": { - "env_var_name": "DEFAULT_MIN_BATCH_SIZE", - "value": "1", - "title": "Default Starting Batch Size", - "description": "Batch size for the first request, which will be multiplied by the growth factor every subsequent request", - "required": false, - "type": "number" - }, - "DEFAULT_BATCH_SIZE_GROWTH_FACTOR": { - "env_var_name": "DEFAULT_BATCH_SIZE_GROWTH_FACTOR", - "value": "3", - "title": "Default Batch Size Growth Factor", - "description": "Growth factor for dynamic batch size", - "required": false, - "type": "number" - }, - "RAW_OPENAI_OUTPUT": { - "env_var_name": "RAW_OPENAI_OUTPUT", - "value": true, - "title": "Raw OpenAI Output", - "description": "Raw OpenAI output instead of just the text", - "required": false, - "type": "toggle" - }, - "OPENAI_RESPONSE_ROLE": { - "env_var_name": "OPENAI_RESPONSE_ROLE", - "value": "assistant", - "title": "OpenAI Response Role", - "description": "Role of the LLM's Response in OpenAI Chat Completions", - "required": false, - "type": "text" - }, - "OPENAI_SERVED_MODEL_NAME_OVERRIDE": { - "env_var_name": "OPENAI_SERVED_MODEL_NAME_OVERRIDE", - "value": "", - "title": "OpenAI Served Model Name Override", - "description": "Overrides the name of the served model from model repo/path to specified name, which you will then be able to use the value for the `model` parameter when making OpenAI requests", - "required": false, - "type": "text" - }, - "MAX_CONCURRENCY": { - "env_var_name": "MAX_CONCURRENCY", - "value": "300", - "title": "Max Concurrency", - "description": "Max concurrent requests per worker. vLLM has an internal queue, so you don't have to worry about limiting by VRAM, this is for improving scaling/load balancing efficiency", - "required": false, - "type": "number" - }, - "MODEL_REVISION": { - "env_var_name": "MODEL_REVISION", - "value": "", - "title": "Model Revision", - "description": "Model revision (branch) to load", - "required": false, - "type": "text" - }, - "BASE_PATH": { - "env_var_name": "BASE_PATH", - "value": "/runpod-volume", - "title": "Base Path", - "description": "Storage directory for Huggingface cache and model", - "required": false, - "type": "text" - }, - "DISABLE_LOG_REQUESTS": { - "env_var_name": "DISABLE_LOG_REQUESTS", - "value": true, - "title": "Disable Log Requests", - "description": "Enables or disables vLLM request logging", - "required": false, - "type": "toggle" - }, - "ENABLE_AUTO_TOOL_CHOICE": { - "env_var_name": "ENABLE_AUTO_TOOL_CHOICE", - "value": false, - "title": "Enable Auto Tool Choice", - "description": "Enables or disables auto tool choice", - "required": false, - "type": "toggle" - }, - "TOOL_CALL_PARSER": { - "env_var_name": "TOOL_CALL_PARSER", - "value": "", - "title": "Tool Call Parser", - "description": "Tool call parser", - "required": false, - "type": "select", - "options": [ - { "value": "", "label": "None" }, - { "value": "hermes", "label": "Hermes" }, - { "value": "mistral", "label": "Mistral" }, - { "value": "llama3_json", "label": "Llama3 JSON" }, - { "value": "pythonic", "label": "Pythonic" }, - { "value": "internlm", "label": "InternLM" } - ] - } -} -}