Bump vLLM to 0.17.0

bug: fix rope scaling to be forward compatible from hf_overrides

.runpod/README.md

@@ -28,6 +28,8 @@ All behaviour is controlled through environment variables:
 | `OPENAI_SERVED_MODEL_NAME_OVERRIDE` | Override served model name in API                 |                     | String                                                             |
 | `MAX_CONCURRENCY`                   | Maximum concurrent requests                       | 300                 | Integer                                                            |
 
+**Pass any vLLM engine arg** not listed above by setting an env var with the **UPPERCASED** field name (e.g. `MAX_MODEL_LEN=4096`, `ENABLE_CHUNKED_PREFILL=true`). The worker auto-discovers all `AsyncEngineArgs` fields from env. See the [vLLM engine args docs](https://docs.vllm.ai/en/latest/configuration/engine_args) for all available options.
+
 For complete configuration options, see the [full configuration documentation](https://github.com/runpod-workers/worker-vllm/blob/main/docs/configuration.md).
 
 ## API Usage

.runpod/hub.json

@@ -187,6 +187,7 @@
           "name": "Max Model Length",
           "type": "number",
           "description": "Model context length.",
+          "default": null,
           "advanced": true
         }
       },
@@ -206,7 +207,8 @@
               "value": "mp"
             }
           ],
-          "advanced": true
+          "advanced": true,
+          "default": "mp"
         }
       },
       {
@@ -278,21 +280,13 @@
           "advanced": true
         }
       },
-      {
-        "key": "NUM_GPU_BLOCKS_OVERRIDE",
-        "input": {
-          "name": "Num GPU Blocks Override",
-          "type": "number",
-          "description": "If specified, ignore GPU profiling result and use this number of GPU blocks.",
-          "advanced": true
-        }
-      },
       {
         "key": "MAX_NUM_BATCHED_TOKENS",
         "input": {
           "name": "Max Num Batched Tokens",
           "type": "number",
           "description": "Maximum number of batched tokens per iteration.",
+          "default": null,
           "advanced": true
         }
       },
@@ -490,6 +484,61 @@
           "advanced": true
         }
       },
+      {
+        "key": "SPECULATIVE_CONFIG",
+        "input": {
+          "name": "Speculative Config (JSON)",
+          "type": "string",
+          "description": "Full speculative decoding configuration as a JSON string. Overrides individual speculative env vars.",
+          "advanced": true
+        }
+      },
+      {
+        "key": "SPECULATIVE_METHOD",
+        "input": {
+          "name": "Speculative Method",
+          "type": "string",
+          "description": "Speculative decoding method to use.",
+          "options": [
+            { "label": "None", "value": "" },
+            { "label": "Draft Model", "value": "draft_model" },
+            { "label": "N-gram", "value": "ngram" },
+            { "label": "EAGLE", "value": "eagle" },
+            { "label": "EAGLE3", "value": "eagle3" },
+            { "label": "Medusa", "value": "medusa" },
+            { "label": "MLP Speculator", "value": "mlp_speculator" }
+          ],
+          "default": "",
+          "advanced": true
+        }
+      },
+      {
+        "key": "SPECULATIVE_MODEL",
+        "input": {
+          "name": "Speculative Model",
+          "type": "string",
+          "description": "The name of the draft model to be used in speculative decoding.",
+          "advanced": true
+        }
+      },
+      {
+        "key": "NUM_SPECULATIVE_TOKENS",
+        "input": {
+          "name": "Num Speculative Tokens",
+          "type": "number",
+          "description": "The number of speculative tokens to sample from the draft model.",
+          "advanced": true
+        }
+      },
+      {
+        "key": "NGRAM_PROMPT_LOOKUP_MAX",
+        "input": {
+          "name": "Ngram Prompt Lookup Max",
+          "type": "number",
+          "description": "Max size of window for ngram prompt lookup in speculative decoding.",
+          "advanced": true
+        }
+      },
       {
         "key": "MODEL_LOADER_EXTRA_CONFIG",
         "input": {

Dockerfile

@@ -1,13 +1,13 @@
-FROM nvidia/cuda:12.8.0-base-ubuntu22.04 
+FROM nvidia/cuda:12.9.1-base-ubuntu22.04 
 
 RUN apt-get update -y \
     && apt-get install -y python3-pip
 
-RUN ldconfig /usr/local/cuda-12.8/compat/
+RUN ldconfig /usr/local/cuda-12.9/compat/
 
-# Install vLLM with FlashInfer - use CUDA 12.8 PyTorch wheels (compatible with vLLM 0.15.0)
+# Install vLLM with FlashInfer from the CUDA 12.9 wheel index.
 RUN python3 -m pip install --upgrade pip && \
-    python3 -m pip install "vllm[flashinfer]==0.15.0" --extra-index-url https://download.pytorch.org/whl/cu128
+    python3 -m pip install "vllm[flashinfer]==0.17.0" --extra-index-url https://download.pytorch.org/whl/cu129
 
 
 
@@ -23,6 +23,7 @@ ARG BASE_PATH="/runpod-volume"
 ARG QUANTIZATION=""
 ARG MODEL_REVISION=""
 ARG TOKENIZER_REVISION=""
+ARG VLLM_NIGHTLY="false"
 
 ENV MODEL_NAME=$MODEL_NAME \
     MODEL_REVISION=$MODEL_REVISION \
@@ -44,6 +45,11 @@ ENV MODEL_NAME=$MODEL_NAME \
 
 ENV PYTHONPATH="/:/vllm-workspace"
 
+RUN if [ "${VLLM_NIGHTLY}" = "true" ]; then \
+    pip install -U vllm --pre --index-url https://pypi.org/simple --extra-index-url https://wheels.vllm.ai/nightly && \
+    apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* && \
+    pip install git+https://github.com/huggingface/transformers.git; \
+fi
 
 COPY src /src
 RUN --mount=type=secret,id=HF_TOKEN,required=false \

README.md

@@ -59,6 +59,16 @@ Configure worker-vllm using environment variables:
 | `OPENAI_SERVED_MODEL_NAME_OVERRIDE` | Override served model name in API                 |                     | String                                                             |
 | `MAX_CONCURRENCY`                   | Maximum concurrent requests                       | 30                  | Integer                                                            |
 
+**Pass any vLLM engine arg** not listed above by setting an environment variable with the **UPPERCASED** field name (same names vLLM uses). The worker auto-discovers all `AsyncEngineArgs` fields from env. For example:
+
+| Environment Variable      | vLLM Engine Arg          | Example Value |
+| ------------------------- | ------------------------ | ------------- |
+| `MAX_MODEL_LEN`           | `max_model_len`          | `4096`        |
+| `ENFORCE_EAGER`           | `enforce_eager`          | `true`        |
+| `ENABLE_CHUNKED_PREFILL`  | `enable_chunked_prefill` | `true`        |
+
+Any env var whose name matches a valid `AsyncEngineArgs` field (uppercased) is applied automatically. Backward-compat aliases: `MODEL_NAME`, `TOKENIZER_NAME`, `MAX_CONTEXT_LEN_TO_CAPTURE`. This lets you configure any vLLM option without waiting for explicit worker support.
+
 For the complete list of all available environment variables, examples, and detailed descriptions: **[Configuration](docs/configuration.md)**
 
 ## Option 2: Build Docker Image with Model Inside
@@ -80,6 +90,7 @@ To build an image with the model baked in, you must specify the following docker
   - `WORKER_CUDA_VERSION`: `12.1.0` (`12.1.0` is recommended for optimal performance).
   - `TOKENIZER_NAME`: Tokenizer repository if you would like to use a different tokenizer than the one that comes with the model. (default: `None`, which uses the model's tokenizer)
   - `TOKENIZER_REVISION`: Tokenizer revision to load (default: `main`).
+  - `VLLM_NIGHTLY`: Set to `true` to replace the pinned vLLM release with the latest nightly build and the latest `transformers` from source. Useful for testing unreleased vLLM features. (default: `false`)
 
 For the remaining settings, you may apply them as environment variables when running the container. Supported environment variables are listed in the [Environment Variables](#environment-variables) section.
 
@@ -89,6 +100,20 @@ For the remaining settings, you may apply them as environment variables when run
 docker build -t username/image:tag --build-arg MODEL_NAME="openchat/openchat_3.5" --build-arg BASE_PATH="/models" .
 ```
 
+### Example: Building with vLLM Nightly
+
+To use the latest unreleased vLLM build (installs from the nightly wheel index and `transformers` from source):
+
+```bash
+docker build -t username/image:tag --build-arg VLLM_NIGHTLY=true .
+```
+
+You can combine it with other arguments:
+
+```bash
+docker build -t username/image:tag --build-arg VLLM_NIGHTLY=true --build-arg MODEL_NAME="meta-llama/Llama-3.1-8B-Instruct" --build-arg BASE_PATH="/models" .
+```
+
 ### (Optional) Including Huggingface Token
 
 If the model you would like to deploy is private or gated, you will need to include it during build time as a Docker secret, which will protect it from being exposed in the image and on DockerHub.

docs/configuration.md

@@ -60,22 +60,32 @@ Complete guide to all environment variables and configuration options for worker
 
 ## Speculative Decoding Settings
 
-| Variable                                         | Default             | Type/Choices                                        | Description                                                                               |
+Speculative decoding can be configured in two ways:
-| ------------------------------------------------ | ------------------- | --------------------------------------------------- | ----------------------------------------------------------------------------------------- |
-| `SCHEDULER_DELAY_FACTOR`                         | 0.0                 | `float`                                             | Apply a delay before scheduling next prompt.                                              |
-| `ENABLE_CHUNKED_PREFILL`                         | False               | `bool`                                              | Enable chunked prefill requests.                                                          |
-| `SPECULATIVE_MODEL`                              | None                | `str`                                               | The name of the draft model to be used in speculative decoding.                           |
-| `NUM_SPECULATIVE_TOKENS`                         | None                | `int`                                               | The number of speculative tokens to sample from the draft model.                          |
-| `SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE`         | None                | `int`                                               | Number of tensor parallel replicas for the draft model.                                   |
-| `SPECULATIVE_MAX_MODEL_LEN`                      | None                | `int`                                               | The maximum sequence length supported by the draft model.                                 |
-| `SPECULATIVE_DISABLE_BY_BATCH_SIZE`              | None                | `int`                                               | Disable speculative decoding if the number of enqueue requests is larger than this value. |
-| `NGRAM_PROMPT_LOOKUP_MAX`                        | None                | `int`                                               | Max size of window for ngram prompt lookup in speculative decoding.                       |
-| `NGRAM_PROMPT_LOOKUP_MIN`                        | None                | `int`                                               | Min size of window for ngram prompt lookup in speculative decoding.                       |
-| `SPEC_DECODING_ACCEPTANCE_METHOD`                | 'rejection_sampler' | ['rejection_sampler', 'typical_acceptance_sampler'] | Specify the acceptance method for draft token verification in speculative decoding.       |
-| `TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD` | None                | `float`                                             | Set the lower bound threshold for the posterior probability of a token to be accepted.    |
-| `TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA`     | None                | `float`                                             | A scaling factor for the entropy-based threshold for token acceptance.                    |
 
-## System Performance Settings
+### Option 1: JSON Configuration
+
+Set `SPECULATIVE_CONFIG` to a JSON string with your full speculative decoding configuration:
+
+```bash
+SPECULATIVE_CONFIG='{"method": "ngram", "num_speculative_tokens": 5, "prompt_lookup_max": 4}'
+```
+
+### Option 2: Individual Environment Variables
+
+| Variable                                 | Default | Type/Choices                                                       | Description                                                                               |
+| ---------------------------------------- | ------- | ------------------------------------------------------------------ | ----------------------------------------------------------------------------------------- |
+| `SPECULATIVE_METHOD`                     | None    | ['draft_model', 'ngram', 'eagle', 'eagle3', 'medusa', 'mlp_speculator'] | Speculative decoding method to use.                                                       |
+| `SPECULATIVE_MODEL`                      | None    | `str`                                                              | The name of the draft model to be used in speculative decoding.                           |
+| `NUM_SPECULATIVE_TOKENS`                 | None    | `int`                                                              | The number of speculative tokens to sample from the draft model.                          |
+| `SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE` | None    | `int`                                                              | Number of tensor parallel replicas for the draft model.                                   |
+| `SPECULATIVE_MAX_MODEL_LEN`              | None    | `int`                                                              | The maximum sequence length supported by the draft model.                                 |
+| `SPECULATIVE_DISABLE_BY_BATCH_SIZE`      | None    | `int`                                                              | Disable speculative decoding if the number of enqueue requests is larger than this value. |
+| `NGRAM_PROMPT_LOOKUP_MAX`                | None    | `int`                                                              | Max size of window for ngram prompt lookup in speculative decoding.                       |
+| `NGRAM_PROMPT_LOOKUP_MIN`                | None    | `int`                                                              | Min size of window for ngram prompt lookup in speculative decoding.                       |
+
+If `SPECULATIVE_CONFIG` is set, it takes priority over individual env vars. When using individual env vars without `SPECULATIVE_METHOD`, the method is auto-detected from the model name or configuration.
+
+## Scheduling & Performance Settings
 
 | Variable                       | Default | Type/Choices    | Description                                                                                                                         |
 | ------------------------------ | ------- | --------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
@@ -146,6 +156,29 @@ The way this works is that the first request will have a batch size of `DEFAULT_
 | `DISABLE_LOGGING_REQUEST`   | False   | `bool`  | Disable logging requests.                                                                                                                              |
 | `MAX_LOG_LEN`               | None    | `int`   | Max number of prompt characters or prompt ID numbers being printed in log.                                                                             |
 
+## UPPERCASED env vars: Pass any engine arg
+
+Any vLLM `AsyncEngineArgs` field can be set via an environment variable using the **UPPERCASED** field name (the same names vLLM uses). The worker auto-discovers all fields from env — no prefix.
+
+**Format:** `<FIELD_NAME_UPPERCASED>=<value>` (e.g. `MAX_MODEL_LEN=4096`)
+
+**Examples:**
+
+| Environment Variable     | vLLM Engine Arg          | Value Example |
+| ------------------------ | ------------------------ | ------------- |
+| `MAX_MODEL_LEN`          | `max_model_len`          | `4096`        |
+| `ENFORCE_EAGER`          | `enforce_eager`          | `true`        |
+| `ENABLE_CHUNKED_PREFILL` | `enable_chunked_prefill` | `true`        |
+| `NUM_SCHEDULER_STEPS`    | `num_scheduler_steps`    | `8`           |
+| `TOKENIZER_POOL_SIZE`    | `tokenizer_pool_size`   | `4`           |
+
+**Backward-compat aliases:** `MODEL_NAME` → `model`, `TOKENIZER_NAME` → `tokenizer`, `MAX_CONTEXT_LEN_TO_CAPTURE` → `max_seq_len_to_capture`, `MODEL_REVISION` → `revision`.
+
+**Notes:**
+- Only valid `AsyncEngineArgs` fields are applied. Unknown keys are silently ignored.
+- Values are automatically cast to the correct type (`int`, `float`, `bool`, `str`, or JSON for `dict`/`list`/`tuple`).
+- For a full list of available engine args, see the [vLLM AsyncEngineArgs documentation](https://docs.vllm.ai/en/latest/configuration/engine_args/).
+
 ## Docker Build Arguments
 
 These variables are used when building custom Docker images with models baked in:

src/engine.py

@@ -175,7 +175,7 @@ class vLLMEngine:
 class OpenAIvLLMEngine(vLLMEngine):
     def __init__(self, vllm_engine):
         super().__init__(vllm_engine)
-        self.served_model_name = os.getenv("OPENAI_SERVED_MODEL_NAME_OVERRIDE") or self.engine_args.model
+        self.served_model_name = os.getenv("OPENAI_SERVED_MODEL_NAME_OVERRIDE") or self.engine_args.served_model_name or self.engine_args.model
         self.response_role = os.getenv("OPENAI_RESPONSE_ROLE") or "assistant"
         self.lora_adapters = self._load_lora_adapters()
 
@@ -233,7 +233,7 @@ class OpenAIvLLMEngine(vLLMEngine):
     async def _initialize_engines(self):
         self.model_config = self.llm.model_config
         self.base_model_paths = [
-            BaseModelPath(name=self.engine_args.model, model_path=self.engine_args.model)
+            BaseModelPath(name=self.served_model_name, model_path=self.engine_args.model)
         ]
 
         self.serving_models = OpenAIServingModels(

src/engine_args.py

@@ -1,124 +1,335 @@
 import os
 import json
 import logging
+from typing import get_origin, get_args
 from torch.cuda import device_count
 from vllm import AsyncEngineArgs
 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
 from src.utils import convert_limit_mm_per_prompt
 
-RENAME_ARGS_MAP = {
+# Backward-compat: env var names users already know → engine arg name
+ENV_ALIASES = {
     "MODEL_NAME": "model",
     "MODEL_REVISION": "revision",
     "TOKENIZER_NAME": "tokenizer",
-    "MAX_CONTEXT_LEN_TO_CAPTURE": "max_seq_len_to_capture"
 }
 
+# Literal defaults from original worker (used when env/local do not set a value)
 DEFAULT_ARGS = {
-    "disable_log_stats": os.getenv('DISABLE_LOG_STATS', 'False').lower() == 'true',
+    "disable_log_stats": False,
-    # disable_log_requests is deprecated, use enable_log_requests instead
+    "enable_log_requests": False,
-    "enable_log_requests": os.getenv('ENABLE_LOG_REQUESTS', 'False').lower() == 'true',
+    "gpu_memory_utilization": 0.95,
-    "gpu_memory_utilization": float(os.getenv('GPU_MEMORY_UTILIZATION', 0.95)),
+    "pipeline_parallel_size": 1,
-    "pipeline_parallel_size": int(os.getenv('PIPELINE_PARALLEL_SIZE', 1)),
+    "tensor_parallel_size": 1,
-    "tensor_parallel_size": int(os.getenv('TENSOR_PARALLEL_SIZE', 1)),
+    "skip_tokenizer_init": False,
-    "served_model_name": os.getenv('SERVED_MODEL_NAME', None),
+    "tokenizer_mode": "auto",
-    "tokenizer": os.getenv('TOKENIZER', None),
+    "trust_remote_code": False,
-    "skip_tokenizer_init": os.getenv('SKIP_TOKENIZER_INIT', 'False').lower() == 'true',
+    "load_format": "auto",
-    "tokenizer_mode": os.getenv('TOKENIZER_MODE', 'auto'),
+    "dtype": "auto",
-    "trust_remote_code": os.getenv('TRUST_REMOTE_CODE', 'False').lower() == 'true',
+    "kv_cache_dtype": "auto",
-    "download_dir": os.getenv('DOWNLOAD_DIR', None),
+    "seed": 0,
-    "load_format": os.getenv('LOAD_FORMAT', 'auto'),
+    "worker_use_ray": False,
-    "config_format": os.getenv('CONFIG_FORMAT', 'auto'),
+    "block_size": 16,
-    "dtype": os.getenv('DTYPE', 'auto'),
+    "enable_prefix_caching": False,
-    "kv_cache_dtype": os.getenv('KV_CACHE_DTYPE', 'auto'),
+    "disable_sliding_window": False,
-    "quantization_param_path": os.getenv('QUANTIZATION_PARAM_PATH', None),
+    "swap_space": 4,
-    "seed": int(os.getenv('SEED', 0)),
+    "cpu_offload_gb": 0,
-    "max_model_len": int(os.getenv('MAX_MODEL_LEN', 0)) or None,
+    "max_num_seqs": 256,
-    "worker_use_ray": os.getenv('WORKER_USE_RAY', 'False').lower() == 'true',
+    "max_logprobs": 20,
-    "distributed_executor_backend": os.getenv('DISTRIBUTED_EXECUTOR_BACKEND', None),
+    "enforce_eager": False,
-    "max_parallel_loading_workers": int(os.getenv('MAX_PARALLEL_LOADING_WORKERS', 0)) or None,
+    "max_seq_len_to_capture": 8192,
-    "block_size": int(os.getenv('BLOCK_SIZE', 16)),
+    "disable_custom_all_reduce": False,
-    "enable_prefix_caching": os.getenv('ENABLE_PREFIX_CACHING', 'False').lower() == 'true',
+    "tokenizer_pool_size": 0,
-    "disable_sliding_window": os.getenv('DISABLE_SLIDING_WINDOW', 'False').lower() == 'true',
+    "tokenizer_pool_type": "ray",
-    # attention_backend replaces deprecated VLLM_ATTENTION_BACKEND env var
+    "enable_lora": False,
-    "attention_backend": os.getenv('ATTENTION_BACKEND', None),
+    "max_loras": 1,
-    # Enabled by default for improved throughput. Set to False to disable if experiencing issues
+    "max_lora_rank": 16,
-    "async_scheduling": None if os.getenv('ASYNC_SCHEDULING') is None else os.getenv('ASYNC_SCHEDULING', 'True').lower() == 'true',
+    "enable_prompt_adapter": False,
-    # Controls how often to yield streaming results
+    "max_prompt_adapters": 1,
-    "stream_interval": int(os.getenv('STREAM_INTERVAL', 1)),
+    "max_prompt_adapter_token": 0,
-    "swap_space": int(os.getenv('SWAP_SPACE', 4)),  # GiB
+    "fully_sharded_loras": False,
-    "cpu_offload_gb": int(os.getenv('CPU_OFFLOAD_GB', 0)),  # GiB
+    "lora_extra_vocab_size": 256,
-    # vLLM defaults None to 2048; keep 0 as None to let vLLM auto-calculate
+    "lora_dtype": "auto",
-    "max_num_batched_tokens": int(os.getenv('MAX_NUM_BATCHED_TOKENS', 0)) or None,
+    "device": "auto",
-    "max_num_seqs": int(os.getenv('MAX_NUM_SEQS', 256)),
+    "ray_workers_use_nsight": False,
-    "max_logprobs": int(os.getenv('MAX_LOGPROBS', 20)),  # Default value for OpenAI Chat Completions API
+    "num_lookahead_slots": 0,
-    "revision": os.getenv('REVISION', None),
+    "scheduler_delay_factor": 0.0,
-    "code_revision": os.getenv('CODE_REVISION', None),
+    "guided_decoding_backend": "outlines",
-    "rope_scaling": os.getenv('ROPE_SCALING', None),
+    "spec_decoding_acceptance_method": "rejection_sampler",
-    "rope_theta": float(os.getenv('ROPE_THETA', 0)) or None,
+    "stream_interval": 1,
-    "tokenizer_revision": os.getenv('TOKENIZER_REVISION', None),
+
-    "quantization": os.getenv('QUANTIZATION', None),
-    "enforce_eager": os.getenv('ENFORCE_EAGER', 'False').lower() == 'true',
-    "max_context_len_to_capture": int(os.getenv('MAX_CONTEXT_LEN_TO_CAPTURE', 0)) or None,
-    "max_seq_len_to_capture": int(os.getenv('MAX_SEQ_LEN_TO_CAPTURE', 8192)),
-    "disable_custom_all_reduce": os.getenv('DISABLE_CUSTOM_ALL_REDUCE', 'False').lower() == 'true',
-    "tokenizer_pool_size": int(os.getenv('TOKENIZER_POOL_SIZE', 0)),
-    "tokenizer_pool_type": os.getenv('TOKENIZER_POOL_TYPE', 'ray'),
-    "tokenizer_pool_extra_config": os.getenv('TOKENIZER_POOL_EXTRA_CONFIG', None),
-    "enable_lora": os.getenv('ENABLE_LORA', 'False').lower() == 'true',
-    "max_loras": int(os.getenv('MAX_LORAS', 1)),
-    "max_lora_rank": int(os.getenv('MAX_LORA_RANK', 16)),
-    "enable_prompt_adapter": os.getenv('ENABLE_PROMPT_ADAPTER', 'False').lower() == 'true',
-    "max_prompt_adapters": int(os.getenv('MAX_PROMPT_ADAPTERS', 1)),
-    "max_prompt_adapter_token": int(os.getenv('MAX_PROMPT_ADAPTER_TOKEN', 0)),
-    "fully_sharded_loras": os.getenv('FULLY_SHARDED_LORAS', 'False').lower() == 'true',
-    "lora_extra_vocab_size": int(os.getenv('LORA_EXTRA_VOCAB_SIZE', 256)),
-    "long_lora_scaling_factors": tuple(map(float, os.getenv('LONG_LORA_SCALING_FACTORS', '').split(','))) if os.getenv('LONG_LORA_SCALING_FACTORS') else None,
-    "lora_dtype": os.getenv('LORA_DTYPE', 'auto'),
-    "max_cpu_loras": int(os.getenv('MAX_CPU_LORAS', 0)) or None,
-    "device": os.getenv('DEVICE', 'auto'),
-    "ray_workers_use_nsight": os.getenv('RAY_WORKERS_USE_NSIGHT', 'False').lower() == 'true',
-    "num_gpu_blocks_override": int(os.getenv('NUM_GPU_BLOCKS_OVERRIDE', 0)) or None,
-    "num_lookahead_slots": int(os.getenv('NUM_LOOKAHEAD_SLOTS', 0)),
-    "model_loader_extra_config": os.getenv('MODEL_LOADER_EXTRA_CONFIG', None),
-    "ignore_patterns": os.getenv('IGNORE_PATTERNS', None),
-    "preemption_mode": os.getenv('PREEMPTION_MODE', None),
-    "scheduler_delay_factor": float(os.getenv('SCHEDULER_DELAY_FACTOR', 0.0)),
-    "enable_chunked_prefill": os.getenv('ENABLE_CHUNKED_PREFILL', None),
-    "guided_decoding_backend": os.getenv('GUIDED_DECODING_BACKEND', 'outlines'),
-    "speculative_model": os.getenv('SPECULATIVE_MODEL', None),
-    "speculative_draft_tensor_parallel_size": int(os.getenv('SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE', 0)) or None,
-    "enable_expert_parallel": bool(os.getenv('ENABLE_EXPERT_PARALLEL', 'False').lower() == 'true'),
-    "num_speculative_tokens": int(os.getenv('NUM_SPECULATIVE_TOKENS', 0)) or None,
-    "speculative_max_model_len": int(os.getenv('SPECULATIVE_MAX_MODEL_LEN', 0)) or None,
-    "speculative_disable_by_batch_size": int(os.getenv('SPECULATIVE_DISABLE_BY_BATCH_SIZE', 0)) or None,
-    "ngram_prompt_lookup_max": int(os.getenv('NGRAM_PROMPT_LOOKUP_MAX', 0)) or None,
-    "ngram_prompt_lookup_min": int(os.getenv('NGRAM_PROMPT_LOOKUP_MIN', 0)) or None,
-    "spec_decoding_acceptance_method": os.getenv('SPEC_DECODING_ACCEPTANCE_METHOD', 'rejection_sampler'),
-    "typical_acceptance_sampler_posterior_threshold": float(os.getenv('TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD', 0)) or None,
-    "typical_acceptance_sampler_posterior_alpha": float(os.getenv('TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA', 0)) or None,
-    "qlora_adapter_name_or_path": os.getenv('QLORA_ADAPTER_NAME_OR_PATH', None),
-    "disable_logprobs_during_spec_decoding": os.getenv('DISABLE_LOGPROBS_DURING_SPEC_DECODING', None),
-    "otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None),
 }
-limit_mm_env = os.getenv('LIMIT_MM_PER_PROMPT')
-if limit_mm_env is not None:
-    DEFAULT_ARGS["limit_mm_per_prompt"] = convert_limit_mm_per_prompt(limit_mm_env)
 
-def match_vllm_args(args):
-    """Rename args to match vllm by:
-    1. Renaming keys to lower case
-    2. Renaming keys to match vllm
-    3. Filtering args to match vllm's AsyncEngineArgs
 
-    Args:
+def _resolve_field_type(field_type: type) -> type:
-        args (dict): Dictionary of args
+    """Resolve Optional/Union to the concrete type for conversion."""
+    origin = get_origin(field_type)
+    args = get_args(field_type) if hasattr(field_type, "__args__") else ()
+    if origin is not None:
+        # Optional[X] is Union[X, None]; X | None is UnionType
+        non_none = [a for a in args if a is not type(None)]
+        if non_none:
+            return non_none[0]
+    return field_type
 
-    Returns:
+
-        dict: Dictionary of args with renamed keys
+def _convert_env_value_to_field_type(value: str, field_name: str, field_type: type):
+    """Convert env var string to the type expected by AsyncEngineArgs for this field."""
+    val = value.strip() if isinstance(value, str) else value
+    if val in ("", "None", "none"):
+        args = get_args(field_type) if hasattr(field_type, "__args__") else ()
+        if type(None) in (args or ()):
+            return None
+        raise ValueError("empty value not allowed for non-optional field")
+    effective_type = _resolve_field_type(field_type)
+    # bool
+    if effective_type is bool:
+        return str(val).lower() in ("true", "1", "yes", "on")
+    # int
+    if effective_type is int:
+        return int(val)
+    # float
+    if effective_type is float:
+        return float(val)
+    # str
+    if effective_type is str:
+        return str(val)
+    # dict, list, or complex (try JSON)
+    origin = get_origin(effective_type)
+    if effective_type in (dict, list) or origin in (dict, list):
+        try:
+            return json.loads(val)
+        except json.JSONDecodeError:
+            return val
+    # tuple (e.g. long_lora_scaling_factors) — comma-separated or JSON array
+    if effective_type is tuple or origin is tuple:
+        args = get_args(field_type) if hasattr(field_type, "__args__") else ()
+        elem_types = [a for a in args if a is not Ellipsis]
+        elem_type = elem_types[0] if elem_types else str
+        try:
+            parsed = json.loads(val)
+            if isinstance(parsed, list):
+                return tuple(elem_type(x) for x in parsed)
+        except (json.JSONDecodeError, TypeError):
+            pass
+        return tuple(elem_type(x.strip()) for x in str(val).split(",") if x.strip())
+    # Fallback: try int, float, then str
+    try:
+        return int(val)
+    except ValueError:
+        pass
+    try:
+        return float(val)
+    except ValueError:
+        pass
+    return str(val)
+
+
+def _get_args_from_env_auto_discover() -> dict:
+    """Auto-discover engine args from env vars using UPPERCASED field names.
+
+    For every field in AsyncEngineArgs, check os.getenv(FIELD_NAME).
+    E.g. MAX_MODEL_LEN=4096 -> max_model_len=4096.
+    Uses same type conversion as before; supports all vLLM engine args without manual listing.
     """
-    renamed_args = {RENAME_ARGS_MAP.get(k, k): v for k, v in args.items()}
+    args = {}
-    matched_args = {k: v for k, v in renamed_args.items() if k in AsyncEngineArgs.__dataclass_fields__}
+    valid_fields = AsyncEngineArgs.__dataclass_fields__
-    return {k: v for k, v in matched_args.items() if v not in [None, "", "None"]}
+    for field_name, field in valid_fields.items():
+        env_key = field_name.upper()
+        value = os.environ.get(env_key)
+        if value is None:
+            continue
+        try:
+            args[field_name] = _convert_env_value_to_field_type(
+                value, field_name, field.type
+            )
+        except (ValueError, TypeError, json.JSONDecodeError) as e:
+            logging.warning(
+                "Skip env %s=%r: %s", env_key, value, e
+            )
+    return args
+
+
+def _apply_env_aliases(args: dict) -> None:
+    """Apply ENV_ALIASES: if MODEL_NAME etc. are set, set the target engine arg."""
+    valid_fields = AsyncEngineArgs.__dataclass_fields__
+    for alias, target in ENV_ALIASES.items():
+        value = os.environ.get(alias)
+        if value is None or target not in valid_fields:
+            continue
+        try:
+            args[target] = _convert_env_value_to_field_type(
+                value, target, valid_fields[target].type
+            )
+        except (ValueError, TypeError, json.JSONDecodeError) as e:
+            logging.warning("Skip env alias %s=%r: %s", alias, value, e)
+
+def get_speculative_config():
+    """Build speculative decoding configuration from environment variables.
+
+    Supports two modes:
+    1. Full JSON config via SPECULATIVE_CONFIG env var
+    2. Individual env vars for common settings
+    """
+    # Option 1: Full JSON configuration
+    spec_config_json = os.getenv('SPECULATIVE_CONFIG')
+    if spec_config_json:
+        try:
+            config = json.loads(spec_config_json)
+            logging.info(f"Using speculative config from SPECULATIVE_CONFIG: {config}")
+            return config
+        except json.JSONDecodeError as e:
+            logging.error(f"Failed to parse SPECULATIVE_CONFIG JSON: {e}")
+            return None
+
+    # Option 2: Build config from individual environment variables
+    spec_method = os.getenv('SPECULATIVE_METHOD')
+    spec_model = os.getenv('SPECULATIVE_MODEL')
+    _num_spec_tokens = os.getenv('NUM_SPECULATIVE_TOKENS')
+    _ngram_max = os.getenv('NGRAM_PROMPT_LOOKUP_MAX')
+    _ngram_min = os.getenv('NGRAM_PROMPT_LOOKUP_MIN')
+
+    # Convert numeric vars to int so '0' (hub.json default) is treated as unset
+    num_spec_tokens = (int(_num_spec_tokens) or None) if _num_spec_tokens else None
+    ngram_max = (int(_ngram_max) or None) if _ngram_max else None
+    ngram_min = (int(_ngram_min) or None) if _ngram_min else None
+
+    if not any([spec_method, spec_model, ngram_max]):
+        return None
+
+    config = {}
+
+    # Determine method
+    if spec_method:
+        config['method'] = spec_method
+    elif ngram_max and not spec_model:
+        config['method'] = 'ngram'
+    elif spec_model:
+        model_lower = spec_model.lower()
+        if 'eagle3' in model_lower:
+            config['method'] = 'eagle3'
+        elif 'eagle' in model_lower:
+            config['method'] = 'eagle'
+        elif 'medusa' in model_lower:
+            config['method'] = 'medusa'
+        else:
+            config['method'] = 'draft_model'
+
+    if spec_model:
+        config['model'] = spec_model
+    if num_spec_tokens:
+        config['num_speculative_tokens'] = num_spec_tokens
+    if ngram_max:
+        config['prompt_lookup_max'] = ngram_max
+    if ngram_min:
+        config['prompt_lookup_min'] = ngram_min
+
+    draft_tp = os.getenv('SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE')
+    if draft_tp:
+        config['draft_tensor_parallel_size'] = int(draft_tp)
+
+    spec_max_len = os.getenv('SPECULATIVE_MAX_MODEL_LEN')
+    if spec_max_len:
+        config['max_model_len'] = int(spec_max_len)
+
+    disable_batch = os.getenv('SPECULATIVE_DISABLE_BY_BATCH_SIZE')
+    if disable_batch:
+        config['disable_by_batch_size'] = int(disable_batch)
+
+    spec_quant = os.getenv('SPECULATIVE_QUANTIZATION')
+    if spec_quant:
+        config['quantization'] = spec_quant
+
+    spec_revision = os.getenv('SPECULATIVE_MODEL_REVISION')
+    if spec_revision:
+        config['revision'] = spec_revision
+
+    spec_eager = os.getenv('SPECULATIVE_ENFORCE_EAGER')
+    if spec_eager:
+        config['enforce_eager'] = spec_eager.lower() == 'true'
+
+    if config:
+        logging.info(f"Built speculative config from env vars: {config}")
+        return config
+
+    return None
+
+
+def _resolve_max_model_len(model, trust_remote_code=False, revision=None):
+    """Resolve max_model_len from the model's HuggingFace config."""
+    try:
+        from transformers import AutoConfig
+        config = AutoConfig.from_pretrained(
+            model,
+            trust_remote_code=trust_remote_code,
+            revision=revision,
+        )
+        for attr in ('max_position_embeddings', 'n_positions', 'max_seq_len', 'seq_length'):
+            val = getattr(config, attr, None)
+            if val is not None:
+                logging.info(f"Resolved max_model_len={val} from model config ({attr})")
+                return val
+    except Exception as e:
+        logging.warning(f"Could not resolve max_model_len from model config: {e}")
+    return None
+
+
+def _local_args_to_engine_args(local: dict) -> dict:
+    """Map local args (e.g. from /local_model_args.json) to engine arg names and filter."""
+    valid = AsyncEngineArgs.__dataclass_fields__
+    out = {}
+    for k, v in local.items():
+        target = ENV_ALIASES.get(k, k.lower().replace("-", "_"))
+        if target not in valid or v in (None, "", "None"):
+            continue
+        out[target] = v
+    return out
+
+
+def _sanitize_hf_overrides(hf_overrides: dict) -> dict | None:
+    """Strip rope_scaling from hf_overrides sub-configs if vLLM rejects them.
+
+    Older vLLM (<0.7) required explicit mrope rope_scaling in hf_overrides for
+    models like Qwen2-VL. Newer vLLM auto-detects mrope and raises a ValueError
+    in patch_rope_scaling_dict when it finds conflicting rope_type values. Strip
+    the offending rope_scaling so the model loads with its native config.
+    """
+    if not isinstance(hf_overrides, dict):
+        return hf_overrides
+
+    try:
+        from vllm.transformers_utils.config import patch_rope_scaling_dict
+    except ImportError:
+        return hf_overrides
+
+    import copy
+    cleaned = {}
+    changed = False
+    for key, value in hf_overrides.items():
+        if isinstance(value, dict) and "rope_scaling" in value:
+            rope_scaling = value.get("rope_scaling")
+            if isinstance(rope_scaling, dict):
+                try:
+                    patch_rope_scaling_dict(copy.deepcopy(rope_scaling))
+                except (ValueError, Exception) as e:
+                    logging.warning(
+                        "Stripping hf_overrides['%s']['rope_scaling'] because vLLM "
+                        "rejected it (%s). Newer vLLM auto-detects rope scaling from "
+                        "the model config.", key, e
+                    )
+                    stripped = {k: v for k, v in value.items() if k != "rope_scaling"}
+                    cleaned[key] = stripped if stripped else None
+                    changed = True
+                    continue
+        cleaned[key] = value
+
+    if not changed:
+        return hf_overrides
+
+    result = {k: v for k, v in cleaned.items() if v is not None}
+    return result or None
+
+
 def get_local_args():
     """
     Retrieve local arguments from a JSON file.
@@ -141,23 +352,43 @@ def get_local_args():
 
     return local_args
 def get_engine_args():
-    # Start with default args
+    # Start with worker custom defaults (only where we differ from vLLM)
-    args = DEFAULT_ARGS
+    args = dict(DEFAULT_ARGS)
 
-    # Get env args that match keys in AsyncEngineArgs
+    # Auto-discover: every AsyncEngineArgs field from env UPPERCASED (e.g. MAX_MODEL_LEN)
-    args.update(os.environ)
+    args.update(_get_args_from_env_auto_discover())
 
-    # Get local args if model is baked in and overwrite env args
+    # Backward-compat aliases (MODEL_NAME → model, etc.)
-    args.update(get_local_args())
+    _apply_env_aliases(args)
+
+    # Local baked-in model overrides
+    local = get_local_args()
+    if local:
+        args.update(_local_args_to_engine_args(local))
+
+    # Filter to valid engine args and drop sentinel empty values
+    valid_fields = AsyncEngineArgs.__dataclass_fields__
+    args = {
+        k: v for k, v in args.items()
+        if k in valid_fields and v not in (None, "", "None")
+    }
+
+    # Special conversion for limit_mm_per_prompt (e.g. "image=1,video=0")
+    limit_mm_env = os.getenv("LIMIT_MM_PER_PROMPT")
+    if limit_mm_env is not None:
+        args["limit_mm_per_prompt"] = convert_limit_mm_per_prompt(limit_mm_env)
 
     # if args.get("TENSORIZER_URI"): TODO: add back once tensorizer is ready
     #     args["load_format"] = "tensorizer"
     #     args["model_loader_extra_config"] = TensorizerConfig(tensorizer_uri=args["TENSORIZER_URI"], num_readers=None)
     #     logging.info(f"Using tensorized model from {args['TENSORIZER_URI']}")
 
-    
+    if "hf_overrides" in args:
-    # Rename and match to vllm args
+        sanitized = _sanitize_hf_overrides(args["hf_overrides"])
-    args = match_vllm_args(args)
+        if sanitized:
+            args["hf_overrides"] = sanitized
+        else:
+            del args["hf_overrides"]
 
     if args.get("load_format") == "bitsandbytes":
         args["quantization"] = args["load_format"]
@@ -182,11 +413,26 @@ def get_engine_args():
     #     os.environ["VLLM_ATTENTION_BACKEND"] = "FLASHINFER"
     #     logging.info("Using FLASHINFER for gemma-2 model.")
     
-    # When max_num_batched_tokens is None (env var was 0), set to max_model_len
+    # Set max_num_batched_tokens to max_model_len for unlimited batching.
-    # to preserve "unlimited" behavior. vLLM defaults None to 2048.
+    # vLLM defaults max_num_batched_tokens to 2048 when None, which is too low.
-    if args.get("max_num_batched_tokens") is None and args.get("max_model_len") is not None:
+
-        args["max_num_batched_tokens"] = args["max_model_len"]
+    if args.get("max_model_len") == 0:
-        logging.info(f"Setting max_num_batched_tokens to max_model_len ({args['max_model_len']}) for unlimited batching.")
+        args["max_model_len"] = None
+
+    if args.get("max_num_batched_tokens") == 0:
+        args["max_num_batched_tokens"] = None
+
+    if args.get("max_num_batched_tokens") is None:
+        max_model_len = args.get("max_model_len")
+        if max_model_len is None:
+            max_model_len = _resolve_max_model_len(
+                args.get("model"),
+                trust_remote_code=args.get("trust_remote_code", False),
+                revision=args.get("revision"),
+            )
+        if max_model_len is not None:
+            args["max_num_batched_tokens"] = max_model_len
+            logging.info(f"Setting max_num_batched_tokens to {max_model_len}")
     
     # VLLM_ATTENTION_BACKEND is deprecated, migrate to attention_backend
     if os.getenv('VLLM_ATTENTION_BACKEND'):
@@ -207,4 +453,9 @@ def get_engine_args():
         if os.getenv('DISABLE_LOG_REQUESTS', 'False').lower() == 'true':
             args['enable_log_requests'] = False
 
+    # Add speculative decoding configuration if present
+    speculative_config = get_speculative_config()
+    if speculative_config:
+        args["speculative_config"] = speculative_config
+
     return AsyncEngineArgs(**args)

src/utils.py

@@ -6,7 +6,7 @@ from time import time
 
 try:
     from vllm.utils import random_uuid
-    from vllm.entrypoints.openai.engine.protocol import ErrorResponse, RequestResponseMetadata
+    from vllm.entrypoints.openai.engine.protocol import ErrorResponse, ErrorInfo, RequestResponseMetadata
     from vllm import SamplingParams
 except ImportError:
     logging.warning("Error importing vllm, skipping related imports. This is ONLY expected when baking model into docker image from a machine without GPUs")
@@ -87,9 +87,9 @@ class BatchSize:
             self.current_batch_size = min(self.current_batch_size*self.batch_size_growth_factor, self.max_batch_size)
         
 def create_error_response(message: str, err_type: str = "BadRequestError", status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> ErrorResponse:
-    return ErrorResponse(message=message,
+    return ErrorResponse(error=ErrorInfo(message=message,
-                            type=err_type,
+                                         type=err_type,
-                            code=status_code.value)
+                                         code=status_code.value))
     
 def get_int_bool_env(env_var: str, default: bool) -> bool:
     return int(os.getenv(env_var, int(default))) == 1