Merge pull request #311 from runpod-workers/t3code/cd56c69d

fix: serve original model name when HF cache dir is lowercased (#310)

.github/workflows/CI-readme_version_sync.yml

@@ -0,0 +1,71 @@
+name: CI | Sync vLLM version in READMEs
+
+on:
+  push:
+    branches: ["main"]
+    paths:
+      - "Dockerfile"
+
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  sync_version:
+    runs-on: ubuntu-latest
+    name: Check README version matches Dockerfile and update if needed
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Extract vLLM version from Dockerfile and sync READMEs
+        run: |
+          echo "Extracting vLLM version from Dockerfile..."
+          dockerfile_version=$(grep -oP 'vllm(?:\[[\w,]+\])?==\K[\d.]+' Dockerfile | head -1)
+
+          if [ -z "$dockerfile_version" ]; then
+            echo "ERROR: Could not extract vLLM version from Dockerfile."
+            exit 1
+          fi
+          echo "Dockerfile vLLM version: $dockerfile_version"
+          echo "VLLM_VERSION=$dockerfile_version" >> $GITHUB_ENV
+
+          updated=0
+          for readme in README.md .runpod/README.md; do
+            if [ ! -f "$readme" ]; then
+              echo "Skipping $readme (not found)"
+              continue
+            fi
+
+            readme_version=$(grep -oP 'Current vLLM version: \[\K[\d.]+' "$readme" || echo "")
+            echo "$readme current version: ${readme_version:-not found}"
+
+            if [ "$readme_version" = "$dockerfile_version" ]; then
+              echo "$readme is already up to date."
+              continue
+            fi
+
+            echo "Updating $readme from $readme_version to $dockerfile_version..."
+            sed -i "s|Current vLLM version: \[${readme_version}\](https://github.com/vllm-project/vllm/releases/tag/v${readme_version})|Current vLLM version: [${dockerfile_version}](https://github.com/vllm-project/vllm/releases/tag/v${dockerfile_version})|g" "$readme"
+            updated=1
+          done
+
+          echo "UPDATED=$updated" >> $GITHUB_ENV
+
+      - name: Create Pull Request
+        if: env.UPDATED == '1'
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "docs: sync vLLM version to ${{ env.VLLM_VERSION }} in READMEs"
+          title: "docs: sync vLLM version to ${{ env.VLLM_VERSION }} in READMEs"
+          body: |
+            The vLLM version in the Dockerfile has been updated to `${{ env.VLLM_VERSION }}`.
+
+            This PR syncs the version badge/link in:
+            - `README.md`
+            - `.runpod/README.md`
+          branch: docs/sync-vllm-version-${{ env.VLLM_VERSION }}
+          labels: documentation

.github/workflows/release.yml

@@ -1,8 +1,6 @@
 name: Release
 
 on:
-  release:
-    types: [published]
   push:
     tags:
       - "v[0-9]+.[0-9]+.[0-9]+*"

.github/workflows/test.yml

@@ -0,0 +1,32 @@
+name: Tests
+
+on:
+  pull_request:
+    branches:
+      - "**"
+  push:
+    branches:
+      - "main"
+
+permissions:
+  contents: read
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install test dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r tests/requirements.txt
+
+      - name: Run unit tests
+        run: python -m pytest tests -v

.runpod/README.md

@@ -6,7 +6,7 @@ Run LLMs using [vLLM](https://docs.vllm.ai) with an OpenAI-compatible API
 
 [![RunPod](https://api.runpod.io/badge/runpod-workers/worker-vllm)](https://www.runpod.io/console/hub/runpod-workers/worker-vllm)
 
-Current vLLM version: [0.18.1](https://github.com/vllm-project/vllm/releases/tag/v0.16.0)
+Current vLLM version: [0.20.2](https://github.com/vllm-project/vllm/releases/tag/v0.20.2)
 
 ---
 
@@ -29,9 +29,21 @@ All behaviour is controlled through environment variables:
 | `REASONING_PARSER`                  | Parser for reasoning-capable models               |                     | "deepseek_r1", "qwen3", "granite", "hunyuan_a13b"                  |
 | `OPENAI_SERVED_MODEL_NAME_OVERRIDE` | Override served model name in API                 |                     | String                                                             |
 | `MAX_CONCURRENCY`                   | Maximum concurrent requests                       | 300                 | Integer                                                            |
+| `ENFORCE_EAGER`                     |  If True, we will disable CUDA graph and always execute the model in eager mode. If False, we will use CUDA graph and eager execution in hybrid for maximal performance and flexibility. | true | boolean (true or false) | 
 
 **Pass any vLLM engine arg** not listed above by setting an env var with the **UPPERCASED** field name (e.g. `MAX_MODEL_LEN=4096`, `ENABLE_CHUNKED_PREFILL=true`). The worker auto-discovers all `AsyncEngineArgs` fields from env. See the [vLLM engine args docs](https://docs.vllm.ai/en/latest/configuration/engine_args) for all available options.
 
+**Configuration file:** You can also supply a `config.yaml` instead of (or alongside) env vars. Mount it at `/vllm_config.yaml` in the container, or set `VLLM_CONFIG_FILE` to a custom path. Use the same key names as `vllm serve` — hyphens and underscores both work:
+
+```yaml
+model: meta-llama/Llama-3.1-8B-Instruct
+max-model-len: 8192
+gpu-memory-utilization: 0.90
+quantization: awq
+```
+
+Environment variables always override config file values.
+
 For complete configuration options, see the [full configuration documentation](https://github.com/runpod-workers/worker-vllm/blob/main/docs/configuration.md).
 
 ### Specify Transformers Version

.runpod/hub.json

@@ -9,7 +9,7 @@
     "containerDiskInGb": 150,
     "gpuIds": "ADA_80_PRO,AMPERE_80",
     "gpuCount": 1,
-    "allowedCudaVersions": ["12.9", "12.8"],
+    "allowedCudaVersions": ["13.0"],
     "presets": [
       {
         "name": "deepseek-ai/deepseek-r1-distill-llama-8b",
@@ -621,7 +621,7 @@
           "name": "Enforce Eager",
           "type": "boolean",
           "description": "Always use eager-mode PyTorch. If False (0), will use eager mode and CUDA graph in hybrid for maximal performance and flexibility",
-          "default": false,
+          "default": true,
           "advanced": true
         }
       },
@@ -795,6 +795,26 @@
           "default": "",
           "advanced": true
         }
+      },
+      {
+        "key": "PYTORCH_ALLOC_CONF",
+        "input": {
+          "name": "PyTorch Alloc Config",
+          "type": "string",
+          "description": "PyTorch allocation configuration, remove this if you want to use the default configuration",
+          "default": "expandable_segments:True",
+          "advanced": true
+        }
+      },
+      {
+        "key": "VLLM_USE_DEEP_GEMM",
+        "input": {
+          "name": "Use DeepGEMM",
+          "type": "string",
+          "description": "Enable DeepGEMM FP8 kernels (MoE and MQA logits). Set to 1 to enable, 0 to disable. Required for DeepSeek V4 models. Disabled by default — enable on H100/H200 for potential throughput gains. Some GPUs (e.g. H20) may perform better with this off.",
+          "default": "0",
+          "advanced": true
+        }
       }
     ]
   }

.runpod/tests.json

@@ -5,7 +5,7 @@
       "input": {
         "prompt": "Write a short poem about artificial intelligence."
       },
-      "timeout": 30000
+      "timeout": 300000
     },
     {
       "name": "openai_messages_test",
@@ -26,11 +26,11 @@
           "temperature": 0.1
         }
       },
-      "timeout": 30000
+      "timeout": 300000
     }
   ],
   "config": {
-    "gpuTypeId": "NVIDIA GeForce RTX 4090",
+    "gpuTypeId": "NVIDIA L40",
     "gpuCount": 1,
     "env": [
       {
@@ -38,6 +38,6 @@
         "value": "HuggingFaceTB/SmolLM2-135M-Instruct"
       }
     ],
-    "allowedCudaVersions": ["12.9", "12.8", "12.7", "12.6", "12.5"]
+    "allowedCudaVersions": ["13.0"]
   }
 }

Dockerfile

@@ -1,16 +1,17 @@
-FROM nvidia/cuda:12.9.1-base-ubuntu22.04 
+FROM nvidia/cuda:13.0.2-devel-ubuntu22.04
 
 RUN apt-get update -y \
-    && apt-get install -y python3-pip curl \
+    && apt-get install -y python3-pip curl git \
     && curl -LsSf https://astral.sh/uv/install.sh  | sh
 
 ENV PATH="/root/.local/bin:$PATH"
 
-RUN ldconfig /usr/local/cuda-12.9/compat/
+RUN ldconfig /usr/local/cuda-13.0/compat/
 
-# Install vLLM with FlashInfer - use CUDA 12.9 PyTorch wheels
+# Install vLLM with FlashInfer - use CUDA 130 PyTorch wheels
 RUN uv pip install --system "packaging>=24.2" && \
-    uv pip install --system "vllm[flashinfer]==0.18.1" --extra-index-url https://download.pytorch.org/whl/cu129
+    uv pip install --system "vllm[flashinfer]==0.20.2" && \
+    uv pip install --system git+https://github.com/deepseek-ai/DeepGEMM.git@714dd1a4a980f7937a74343d19a8eba4fe321480 --no-build-isolation
 
 # Install additional Python dependencies (after vLLM to avoid PyTorch version conflicts)
 COPY builder/requirements.txt /requirements.txt
@@ -42,7 +43,9 @@ ENV MODEL_NAME=$MODEL_NAME \
     # Prevent rayon thread pool panic in containers where ulimit -u < nproc
     # (tokenizers uses Rust's rayon which tries to spawn threads = CPU cores)
     TOKENIZERS_PARALLELISM=false \
-    RAYON_NUM_THREADS=4
+    RAYON_NUM_THREADS=4 \
+    # Disable DeepGEMM MoE kernels by default; override with VLLM_USE_DEEP_GEMM=1 to enable
+    VLLM_USE_DEEP_GEMM=0
 
 ENV PYTHONPATH="/:/vllm-workspace"
 

README.md

@@ -8,8 +8,8 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https:
 
 ![vLLM worker banner](https://image.runpod.ai/preview/vllm/vllm-banner.png)
 
+Current vLLM version: [0.20.2](https://github.com/vllm-project/vllm/releases/tag/v0.20.2)
 
-Current vLLM version: [0.18.1](https://github.com/vllm-project/vllm/releases/tag/v0.16.0)
 
 > Check out our Load Balancer implementation here: [vLLM Load Balancer](https://github.com/runpod-workers/vllm-loadbalancer-ep)
 
@@ -47,7 +47,7 @@ Current vLLM version: [0.18.1](https://github.com/vllm-project/vllm/releases/tag
 **📦 Docker Image**: `runpod/worker-v1-vllm:<version>`
 
 - **Available Versions**: See [GitHub Releases](https://github.com/runpod-workers/worker-vllm/releases)
-- **CUDA Compatibility**: Requires CUDA >= 12.1
+- **CUDA Compatibility**: Requires CUDA >= 13.0
 
 ### Configuration
 
@@ -78,6 +78,20 @@ Configure worker-vllm using environment variables:
 
 Any env var whose name matches a valid `AsyncEngineArgs` field (uppercased) is applied automatically. Backward-compat aliases: `MODEL_NAME`, `TOKENIZER_NAME`, `MAX_CONTEXT_LEN_TO_CAPTURE`. This lets you configure any vLLM option without waiting for explicit worker support.
 
+### Configuration File (config.yaml)
+
+As an alternative to environment variables, you can supply a `config.yaml` file using the same key names as `vllm serve` (hyphens or underscores both work):
+
+```yaml
+model: meta-llama/Llama-3.1-8B-Instruct
+max-model-len: 8192
+gpu-memory-utilization: 0.90
+quantization: awq
+tensor-parallel-size: 2
+```
+
+Mount the file into the container at `/vllm_config.yaml`, or point to a custom path with the `VLLM_CONFIG_FILE` env var. Environment variables always take precedence over config file values.
+
 For the complete list of all available environment variables, examples, and detailed descriptions: **[Configuration](docs/configuration.md)**
 
 ### Specify Transformers Version

builder/requirements.txt

@@ -1,15 +1,15 @@
 ray
 pandas
 pyarrow
-runpod==1.9.0
+runpod~=1.10.0
 huggingface-hub
-lmcache==0.4.2
+lmcache==0.4.5
 packaging>=24.2
 typing-extensions>=4.8.0
 pydantic
 pydantic-settings
 hf-transfer
-transformers>=4.57.0,<5
+transformers>=5
 bitsandbytes>=0.45.0
-kernels
+kernels<0.15
 torch-c-dlpack-ext

configs/gemma/gemma4_31b_it.yaml

@@ -0,0 +1,11 @@
+model: google/gemma-4-31b-it
+gpu-memory-utilization: 0.95
+max-model-len: 8192
+dtype: auto
+trust-remote-code: true
+quantization: fp8
+kv-cache-dtype: fp8
+enforce-eager: false
+enable-prefix-caching: true
+enable-chunked-prefill: true
+speculative-config: '{"model":"RedHatAI/gemma-4-31B-it-speculator.eagle3","method":"eagle3","num_speculative_tokens":3}'

configs/gpt-oss/gpt_oss_120b.yaml

@@ -0,0 +1,9 @@
+model: openai/gpt-oss-120b
+gpu-memory-utilization: 0.95
+max-model-len: 8192
+dtype: auto
+trust-remote-code: true
+enforce-eager: false
+enable-prefix-caching: true
+enable-chunked-prefill: true
+speculative-config: '{"model":"RedHatAI/gpt-oss-120b-speculator.eagle3","method":"eagle3","num_speculative_tokens":3}'

configs/llama/llama3.1_8b_instruct.yaml

@@ -0,0 +1,10 @@
+model: meta-llama/Llama-3.1-8B-Instruct
+gpu-memory-utilization: 0.95
+max-model-len: 8192
+dtype: auto
+trust-remote-code: true
+quantization: fp8
+kv-cache-dtype: fp8
+enforce-eager: false
+enable-prefix-caching: true
+speculative-config: '{"model":"RedHatAI/Llama-3.1-8B-Instruct-speculator.eagle3","method":"eagle3","num_speculative_tokens":3}'

configs/qwen/qwen3_8b.yaml

@@ -0,0 +1,10 @@
+model: Qwen/Qwen3-8B
+gpu-memory-utilization: 0.95
+max-model-len: 8192
+dtype: auto
+trust-remote-code: true
+quantization: fp8
+kv-cache-dtype: fp8
+enforce-eager: false
+enable-prefix-caching: true
+speculative-config: '{"model":"RedHatAI/Qwen3-8B-speculator.eagle3","method":"eagle3","num_speculative_tokens":3}'

docs/configuration.md

@@ -97,10 +97,13 @@ If `SPECULATIVE_CONFIG` is set, it takes priority over individual env vars. When
 | `MAX_SEQ_LEN_TO_CAPTURE`       | `8192`  | `int`           | Maximum context length covered by CUDA graphs. When a sequence has context length larger than this, we fall back to eager mode.     |
 | `DISABLE_CUSTOM_ALL_REDUCE`    | `0`     | `int`           | Enables or disables custom all reduce.                                                                                              |
 | `ENABLE_EXPERT_PARALLEL`       | `False` | `bool`          | Enable Expert Parallel for MoE models.                                                                                              |
+| `VLLM_USE_DEEP_GEMM`           | `0`     | `str` (`0`/`1`) | Enable DeepGEMM FP8 kernels for MoE and MQA logits computation. Disabled by default. Must be `"0"` or `"1"` — not `true`/`false`. See note below. |
 | `ATTENTION_BACKEND`            | `None`  | `str`           | Attention backend to use (e.g., `FLASH_ATTN`, `FLASHINFER`, `TRITON_FLASH_ATTN`). Replaces deprecated `VLLM_ATTENTION_BACKEND`.     |
 | `ASYNC_SCHEDULING`             | `None`  | `bool`          | Enable async scheduling (overlaps engine scheduling with GPU execution). Default: enabled in vLLM 0.14.0+. Set to `false` to disable. |
 | `STREAM_INTERVAL`              | `1`     | `int`           | Controls how often to yield streaming results. Lower = more frequent updates.                                                        |
 
+> **Note (`VLLM_USE_DEEP_GEMM`):** DeepGEMM is used in two places: MoE weight computation and MQA logits computation. It is necessary for MQA logits computation on supported hardware — required for DeepSeek V4 models. Set `VLLM_USE_DEEP_GEMM=1` to enable. Set `VLLM_USE_DEEP_GEMM=0` to disable the MoE part and fall back to flashinfer/cutlass FP8 kernels. **Value must be `"0"` or `"1"` — not `"true"`/`"false"`.** Some users report better performance with `VLLM_USE_DEEP_GEMM=0`, particularly on H20 GPUs. Disabling it also skips the DeepGEMM warmup phase, reducing cold-start time. Requires CUDA 13.0+ and SM90+ (H100/H200) to use; the library is installed but inactive by default.
+
 ## Tokenizer Settings
 
 | Variable               | Default | Type/Choices                        | Description                                                                                       |

src/engine.py

@@ -1,4 +1,5 @@
 import asyncio
+import inspect
 import json
 import logging
 import os
@@ -7,6 +8,7 @@ from typing import AsyncGenerator, Optional
 
 from dotenv import load_dotenv
 from vllm import AsyncLLMEngine
+from vllm.inputs import TextPrompt
 from vllm.entrypoints.logger import RequestLogger
 from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest, AnthropicMessagesResponse, AnthropicError, AnthropicErrorResponse
 from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
@@ -19,6 +21,7 @@ from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePat
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
 from vllm.entrypoints.openai.responses.protocol import ResponsesRequest, ResponsesResponse
 from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
+from vllm.entrypoints.serve.render.serving import OpenAIServingRender
 
 from constants import DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE_GROWTH_FACTOR, DEFAULT_MAX_CONCURRENCY, DEFAULT_MIN_BATCH_SIZE
 from engine_args import get_engine_args
@@ -29,20 +32,33 @@ class vLLMEngine:
     def __init__(self, engine = None):
         load_dotenv() # For local development
         self.engine_args = get_engine_args()
-        logging.info(f"Engine args: {self.engine_args}")
+
-        
+        if engine is None:
-        # Initialize vLLM engine first
+            ea = self.engine_args
-        self.llm = self._initialize_llm() if engine is None else engine.llm
+            summary = {
-        
+                "model": ea.model,
-        # Only create custom tokenizer wrapper if not using mistral tokenizer mode
+                "dtype": ea.dtype,
-        # For mistral models, let vLLM handle tokenizer initialization
+                "quantization": ea.quantization,
-        if self.engine_args.tokenizer_mode != 'mistral':
+                "max_model_len": ea.max_model_len,
-            self.tokenizer = TokenizerWrapper(self.engine_args.tokenizer or self.engine_args.model, 
+                "tensor_parallel_size": ea.tensor_parallel_size,
-                                              self.engine_args.tokenizer_revision, 
+                "gpu_memory_utilization": ea.gpu_memory_utilization,
-                                              self.engine_args.trust_remote_code)
+            }
+            if ea.tokenizer and ea.tokenizer != ea.model:
+                summary["tokenizer"] = ea.tokenizer
+            logging.info("Engine config: %s", summary)
+            logging.debug("Full engine args: %s", ea)
+
+            self.llm = self._initialize_llm()
+
+            if self.engine_args.tokenizer_mode != 'mistral':
+                self.tokenizer = TokenizerWrapper(self.engine_args.tokenizer or self.engine_args.model,
+                                                  self.engine_args.tokenizer_revision,
+                                                  self.engine_args.trust_remote_code)
+            else:
+                self.tokenizer = None
         else:
-            # For mistral models, we'll get the tokenizer from vLLM later
+            self.llm = engine.llm
-            self.tokenizer = None
+            self.tokenizer = engine.tokenizer
             
         self.max_concurrency = int(os.getenv("MAX_CONCURRENCY", DEFAULT_MAX_CONCURRENCY))
         self.default_batch_size = int(os.getenv("DEFAULT_BATCH_SIZE", DEFAULT_BATCH_SIZE))
@@ -115,7 +131,7 @@ class vLLMEngine:
         if apply_chat_template or isinstance(llm_input, list):
             tokenizer_wrapper = self._get_tokenizer_for_chat_template()
             llm_input = tokenizer_wrapper.apply_chat_template(llm_input)
-        results_generator = self.llm.generate(llm_input, validated_sampling_params, request_id)
+        results_generator = self.llm.generate(TextPrompt(prompt=llm_input), validated_sampling_params, request_id)
         n_responses, n_input_tokens, is_first_output = validated_sampling_params.n, 0, True
         last_output_texts, token_counters = ["" for _ in range(n_responses)], {"batch": 0, "total": 0}
 
@@ -205,19 +221,48 @@ class OpenAIvLLMEngine(vLLMEngine):
             self.raw_openai_output = bool(int(raw_output_env))
 
     def _load_lora_adapters(self):
-        adapters = []
+        lora_modules_env = os.getenv("LORA_MODULES", "")
-        try:
+        if not lora_modules_env:
-            adapters = json.loads(os.getenv("LORA_MODULES", '[]'))
+            return []
-        except Exception as e:
-            logging.info(f"---Initialized adapter json load error: {e}")
 
-        for i, adapter in enumerate(adapters):
+        try:
+            parsed = json.loads(lora_modules_env)
+        except json.JSONDecodeError as e:
+            logging.error(
+                "LORA_MODULES could not be parsed as JSON: %s — no LoRA adapters loaded. Value: %r",
+                e, lora_modules_env,
+            )
+            return []
+
+        # Accept a single adapter dict as well as an array
+        if isinstance(parsed, dict):
+            parsed = [parsed]
+
+        if not isinstance(parsed, list):
+            logging.error(
+                "LORA_MODULES must be a JSON array of adapter objects, got %s — no LoRA adapters loaded.",
+                type(parsed).__name__,
+            )
+            return []
+
+        adapters = []
+        for i, adapter in enumerate(parsed):
             try:
-                adapters[i] = LoRAModulePath(**adapter)
+                adapters.append(LoRAModulePath(**adapter))
-                logging.info(f"---Initialized adapter: {adapter}")
+                logging.info("Loaded LoRA adapter config [%d]: %s", i, adapter)
             except Exception as e:
-                logging.info(f"---Initialized adapter not worked: {e}")
+                logging.error(
-                continue
+                    "Failed to parse LoRA adapter at index %d: %s. Config: %r",
+                    i, e, adapter,
+                )
+
+        if parsed and not adapters:
+            logging.error(
+                "LORA_MODULES specified %d adapter(s) but none could be loaded — "
+                "OpenAI model name lookups for LoRA adapters will fail.",
+                len(parsed),
+            )
+
         return adapters
 
     async def _ensure_engines_initialized(self):
@@ -246,16 +291,32 @@ class OpenAIvLLMEngine(vLLMEngine):
             lora_modules=self.lora_adapters,
         )
         await self.serving_models.init_static_loras()
-        
+
         # Get chat template from vLLM tokenizer if available
         chat_template = None
         if self.tokenizer and hasattr(self.tokenizer, 'tokenizer'):
             chat_template = self.tokenizer.tokenizer.chat_template
-        
+
+        self.openai_serving_render = OpenAIServingRender(
+            model_config=self.llm.model_config,
+            renderer=self.llm.renderer,
+            model_registry=self.serving_models.registry,
+            request_logger=None,
+            chat_template=chat_template,
+            chat_template_content_format="auto",
+            trust_request_chat_template=os.getenv('TRUST_REQUEST_CHAT_TEMPLATE', 'false').lower() == 'true',
+            enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
+            exclude_tools_when_tool_choice_none=os.getenv('EXCLUDE_TOOLS_WHEN_TOOL_CHOICE_NONE', 'false').lower() == 'true',
+            tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None,
+            reasoning_parser=os.getenv('REASONING_PARSER', "") or None,
+            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
+        )
+
         self.chat_engine = OpenAIServingChat(
-            engine_client=self.llm, 
+            engine_client=self.llm,
             models=self.serving_models,
             response_role=self.response_role,
+            openai_serving_render=self.openai_serving_render,
             request_logger=None,
             chat_template=chat_template,
             chat_template_content_format="auto",
@@ -268,20 +329,20 @@ class OpenAIvLLMEngine(vLLMEngine):
             enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
             enable_log_outputs=os.getenv('ENABLE_LOG_OUTPUTS', 'false').lower() == 'true',
-            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
         )
         self.completion_engine = OpenAIServingCompletion(
             engine_client=self.llm,
             models=self.serving_models,
+            openai_serving_render=self.openai_serving_render,
             request_logger=None,
             return_tokens_as_token_ids=os.getenv('RETURN_TOKENS_AS_TOKEN_IDS', 'false').lower() == 'true',
             enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
-            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
         )
         self.responses_engine = OpenAIServingResponses(
             engine_client=self.llm,
             models=self.serving_models,
+            openai_serving_render=self.openai_serving_render,
             request_logger=None,
             chat_template=chat_template,
             chat_template_content_format="auto",
@@ -293,12 +354,12 @@ class OpenAIvLLMEngine(vLLMEngine):
             enable_prompt_tokens_details=os.getenv('ENABLE_PROMPT_TOKENS_DETAILS', 'false').lower() == 'true',
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
             enable_log_outputs=os.getenv('ENABLE_LOG_OUTPUTS', 'false').lower() == 'true',
-            log_error_stack=os.getenv('LOG_ERROR_STACK', 'false').lower() == 'true',
         )
         self.messages_engine = AnthropicServingMessages(
             engine_client=self.llm,
             models=self.serving_models,
             response_role=self.response_role,
+            openai_serving_render=self.openai_serving_render,
             request_logger=None,
             chat_template=chat_template,
             chat_template_content_format="auto",
@@ -310,8 +371,11 @@ class OpenAIvLLMEngine(vLLMEngine):
             enable_force_include_usage=os.getenv('ENABLE_FORCE_INCLUDE_USAGE', 'false').lower() == 'true',
         )
 
-        if hasattr(self.chat_engine, 'warmup'):
+        warmup = getattr(self.chat_engine, 'warmup', None)
-            await self.chat_engine.warmup()
+        if callable(warmup):
+            result = warmup()
+            if inspect.isawaitable(result):
+                await result
 
     async def generate(self, openai_request: JobInput):
         # Ensure engines are ready (no-op if already initialized at startup)

src/engine_args.py

@@ -82,6 +82,16 @@ def _convert_env_value_to_field_type(value: str, field_name: str, field_type: ty
         if type(None) in (args or ()):
             return None
         raise ValueError("empty value not allowed for non-optional field")
+
+    # Union[bool, str, ...]: only coerce to bool for unambiguous literals;
+    # otherwise preserve the string (e.g. hf_token="hf_abc..." must stay a str).
+    if get_origin(field_type) is not None:
+        union_types = [a for a in (get_args(field_type) or ()) if a is not type(None)]
+        if bool in union_types and str in union_types:
+            if str(val).lower() in ("true", "false", "1", "0", "yes", "no", "on", "off"):
+                return str(val).lower() in ("true", "1", "yes", "on")
+            return str(val)
+
     effective_type = _resolve_field_type(field_type)
     # bool
     if effective_type is bool:
@@ -342,6 +352,75 @@ def _sanitize_hf_overrides(hf_overrides: dict) -> dict | None:
     return result or None
 
 
+def _resolve_cached_model_path(model_name: str) -> str:
+    """Return a local snapshot path when the HF cache was stored with lowercase names.
+
+    Some model stores (e.g. RunPod pre-cached volumes) normalize repo IDs to
+    lowercase. HuggingFace Hub stores caches as
+    ``models--{org}--{model}/snapshots/{hash}/`` preserving the original casing,
+    so MODEL_NAME=Qwen/Qwen2.5-Coder-32B-Instruct-AWQ will miss a cache stored
+    as ``models--qwen--qwen2.5-coder-32b-instruct-awq/``.
+
+    If the exact-case cache directory is absent but a lowercase variant exists,
+    the latest snapshot path is returned so vLLM loads from disk rather than
+    attempting a redundant download.
+    """
+    if os.path.isabs(model_name):
+        return model_name
+
+    cache_dir = (
+        os.getenv("HUGGINGFACE_HUB_CACHE")
+        or os.getenv("HF_HOME")
+        or os.path.expanduser("~/.cache/huggingface/hub")
+    )
+
+    folder_name = f"models--{model_name.replace('/', '--')}"
+
+    if os.path.isdir(os.path.join(cache_dir, folder_name)):
+        return model_name
+
+    lower_dir = os.path.join(cache_dir, folder_name.lower())
+    if not os.path.isdir(lower_dir):
+        return model_name
+
+    snapshots_dir = os.path.join(lower_dir, "snapshots")
+    if not os.path.isdir(snapshots_dir):
+        return model_name
+
+    try:
+        snapshots = sorted(os.listdir(snapshots_dir))
+    except OSError:
+        return model_name
+
+    if not snapshots:
+        return model_name
+
+    resolved = os.path.join(snapshots_dir, snapshots[-1])
+    logging.info(
+        "MODEL_NAME %r not found at original casing in HF cache; "
+        "resolved to lowercase cached snapshot at %r",
+        model_name, resolved,
+    )
+    return resolved
+
+
+def _get_args_from_config_file() -> dict:
+    """Load engine args from a vLLM-style config.yaml.
+
+    Checks VLLM_CONFIG_FILE env var, then falls back to /vllm_config.yaml.
+    Keys use the same long-form names as vllm serve (hyphens converted to underscores).
+    """
+    import yaml
+    path = os.getenv("VLLM_CONFIG_FILE", "/vllm_config.yaml")
+    if not os.path.exists(path):
+        return {}
+    with open(path) as f:
+        raw = yaml.safe_load(f) or {}
+    normalized = {k.replace("-", "_"): v for k, v in raw.items()}
+    logging.info("Loaded engine args from config file %s: %s", path, list(normalized.keys()))
+    return normalized
+
+
 def get_local_args():
     """
     Retrieve local arguments from a JSON file.
@@ -367,6 +446,9 @@ def get_engine_args():
     # Start with worker custom defaults (only where we differ from vLLM)
     args = dict(DEFAULT_ARGS)
 
+    # Config file values sit above defaults but below env vars
+    args.update(_get_args_from_config_file())
+
     # Auto-discover: every AsyncEngineArgs field from env UPPERCASED (e.g. MAX_MODEL_LEN)
     args.update(_get_args_from_env_auto_discover())
 
@@ -517,4 +599,15 @@ def get_engine_args():
     if speculative_config:
         args["speculative_config"] = speculative_config
 
+    # Resolve lowercase HF cache paths (FDE-174)
+    if args.get("model"):
+        original_model = args["model"]
+        args["model"] = _resolve_cached_model_path(original_model)
+        # When the model was rewritten to an on-disk snapshot path, keep serving
+        # under the original repo id so the OpenAI API model name does not become
+        # a filesystem path (issue #310). An explicit served_model_name (or the
+        # OPENAI_SERVED_MODEL_NAME_OVERRIDE handled downstream) still wins.
+        if args["model"] != original_model and not args.get("served_model_name"):
+            args["served_model_name"] = original_model
+
     return AsyncEngineArgs(**args)

src/tokenizer.py

@@ -1,10 +1,12 @@
-from transformers import AutoTokenizer
+import logging
 import os
 from typing import Union
 
+from transformers import AutoTokenizer
+
 class TokenizerWrapper:
     def __init__(self, tokenizer_name_or_path, tokenizer_revision, trust_remote_code):
-        print(f"tokenizer_name_or_path: {tokenizer_name_or_path}, tokenizer_revision: {tokenizer_revision}, trust_remote_code: {trust_remote_code}")
+        logging.debug("tokenizer_name_or_path: %s, tokenizer_revision: %s, trust_remote_code: %s", tokenizer_name_or_path, tokenizer_revision, trust_remote_code)
         self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, revision=tokenizer_revision or "main", trust_remote_code=trust_remote_code)
         self.custom_chat_template = os.getenv("CUSTOM_CHAT_TEMPLATE")
         self.has_chat_template = bool(self.tokenizer.chat_template) or bool(self.custom_chat_template)

tests/conftest.py

@@ -0,0 +1,105 @@
+"""Shared test fixtures.
+
+``src/engine_args.py`` hard-imports ``vllm`` (and a tensorizer submodule) and
+``torch.cuda``. Both are only installed inside the GPU Docker image, so when the
+tests run on a machine without them we install lightweight stubs. When the real
+packages *are* available (e.g. CI inside the worker image) the stubs are skipped
+and the real ones are used instead.
+"""
+
+import sys
+import types
+from dataclasses import dataclass
+from typing import Optional, Union, List
+
+
+def _install_torch_stub():
+    try:
+        import torch  # noqa: F401
+        return  # real torch present, nothing to stub
+    except Exception:
+        pass
+
+    torch = types.ModuleType("torch")
+    cuda = types.ModuleType("torch.cuda")
+    # No GPU in the test environment -> 0 devices (skips tensor-parallel setup).
+    cuda.device_count = lambda: 0
+    torch.cuda = cuda
+    sys.modules["torch"] = torch
+    sys.modules["torch.cuda"] = cuda
+
+
+def _install_vllm_stub():
+    try:
+        import vllm  # noqa: F401
+        return  # real vLLM present, nothing to stub
+    except Exception:
+        pass
+
+    vllm = types.ModuleType("vllm")
+
+    @dataclass
+    class AsyncEngineArgs:
+        # Only the fields the worker actually sets/reads need to exist here;
+        # get_engine_args() filters args down to AsyncEngineArgs.__dataclass_fields__
+        # before construction, so unknown keys are dropped rather than passed.
+        model: Optional[str] = None
+        served_model_name: Optional[Union[str, List[str]]] = None
+        revision: Optional[str] = None
+        tokenizer: Optional[str] = None
+        trust_remote_code: bool = False
+        max_model_len: Optional[int] = None
+        max_num_batched_tokens: Optional[int] = None
+        disable_log_stats: bool = False
+        gpu_memory_utilization: float = 0.9
+        tensor_parallel_size: int = 1
+        max_parallel_loading_workers: Optional[int] = None
+        kv_cache_dtype: Optional[str] = None
+
+    class _Stub:  # pragma: no cover - placeholder for vllm symbols
+        def __init__(self, *args, **kwargs):
+            pass
+
+    vllm.AsyncEngineArgs = AsyncEngineArgs
+    vllm.SamplingParams = _Stub
+    sys.modules["vllm"] = vllm
+
+    # src.utils imports these at module load and uses ErrorResponse as a return
+    # annotation, which Python evaluates eagerly on <3.14 -> must be defined.
+    vllm_utils = types.ModuleType("vllm.utils")
+    vllm_utils.random_uuid = lambda: "stub-uuid"
+    vllm.utils = vllm_utils
+    sys.modules["vllm.utils"] = vllm_utils
+
+    protocol = types.ModuleType("vllm.entrypoints.openai.engine.protocol")
+    protocol.ErrorResponse = _Stub
+    protocol.ErrorInfo = _Stub
+    protocol.RequestResponseMetadata = _Stub
+    for name in (
+        "vllm.entrypoints",
+        "vllm.entrypoints.openai",
+        "vllm.entrypoints.openai.engine",
+    ):
+        sys.modules.setdefault(name, types.ModuleType(name))
+    sys.modules["vllm.entrypoints.openai.engine.protocol"] = protocol
+
+    # vllm.model_executor.model_loader.tensorizer.TensorizerConfig
+    model_executor = types.ModuleType("vllm.model_executor")
+    model_loader = types.ModuleType("vllm.model_executor.model_loader")
+    tensorizer = types.ModuleType("vllm.model_executor.model_loader.tensorizer")
+
+    class TensorizerConfig:  # pragma: no cover - placeholder
+        def __init__(self, *args, **kwargs):
+            pass
+
+    tensorizer.TensorizerConfig = TensorizerConfig
+    model_loader.tensorizer = tensorizer
+    model_executor.model_loader = model_loader
+    vllm.model_executor = model_executor
+    sys.modules["vllm.model_executor"] = model_executor
+    sys.modules["vllm.model_executor.model_loader"] = model_loader
+    sys.modules["vllm.model_executor.model_loader.tensorizer"] = tensorizer
+
+
+_install_torch_stub()
+_install_vllm_stub()

tests/requirements.txt

@@ -0,0 +1,6 @@
+# Test-only dependencies. vllm/torch are stubbed in conftest.py when absent,
+# so the unit tests run on a plain CPU runner without the GPU image.
+pytest>=8,<10
+# get_engine_args() reads a vLLM-style config via PyYAML (a transitive vllm dep
+# at runtime); install it explicitly here since vllm itself is stubbed.
+pyyaml

tests/test_engine_args.py

@@ -0,0 +1,126 @@
+"""Tests for HF cache path resolution and served-model-name decoupling.
+
+Regression coverage for issue #310: when MODEL_NAME is served from a lowercased
+HF cache dir, the cache resolver rewrites engine_args.model to a snapshot path.
+The served model name must stay the original repo id, not the path.
+"""
+
+import os
+
+import pytest
+
+from src import engine_args
+from src.engine_args import _resolve_cached_model_path, get_engine_args
+
+
+MODEL = "Qwen/Qwen3.6-27B-FP8"
+SNAPSHOT_HASH = "e89b16ebf1988b3d6befa7de50abc2d76f26eb09"
+
+
+def _make_cache(root, folder_name, snapshot=SNAPSHOT_HASH):
+    """Create a HF-style ``models--…/snapshots/<hash>/`` dir and return its path."""
+    snap_dir = os.path.join(root, folder_name, "snapshots", snapshot)
+    os.makedirs(snap_dir)
+    return snap_dir
+
+
+def _is_case_sensitive_fs(path):
+    """The lowercase-cache resolution only matters on case-sensitive filesystems.
+
+    On macOS (APFS, case-insensitive by default) ``models--Qwen--…`` and
+    ``models--qwen--…`` collide, so the resolver always sees the exact-case dir
+    as present. Production runs on Linux (case-sensitive), which is what these
+    tests exercise.
+    """
+    probe = os.path.join(path, "CaseProbe")
+    open(probe, "w").close()
+    try:
+        return not os.path.exists(os.path.join(path, "caseprobe"))
+    finally:
+        os.remove(probe)
+
+
+requires_case_sensitive_fs = pytest.mark.skipif(
+    not _is_case_sensitive_fs(os.environ.get("TMPDIR", "/tmp")),
+    reason="lowercase HF cache resolution only applies on case-sensitive filesystems",
+)
+
+
+@pytest.fixture
+def hf_cache(tmp_path, monkeypatch):
+    cache = tmp_path / "hub"
+    cache.mkdir()
+    monkeypatch.setenv("HUGGINGFACE_HUB_CACHE", str(cache))
+    # Make sure HF_HOME does not shadow the explicit cache dir during the test.
+    monkeypatch.delenv("HF_HOME", raising=False)
+    return cache
+
+
+class TestResolveCachedModelPath:
+    def test_exact_case_dir_returns_repo_id(self, hf_cache):
+        _make_cache(str(hf_cache), "models--Qwen--Qwen3.6-27B-FP8")
+        assert _resolve_cached_model_path(MODEL) == MODEL
+
+    def test_no_cache_returns_repo_id(self, hf_cache):
+        assert _resolve_cached_model_path(MODEL) == MODEL
+
+    def test_absolute_path_passthrough(self, hf_cache):
+        path = "/runpod-volume/some/local/model"
+        assert _resolve_cached_model_path(path) == path
+
+    @requires_case_sensitive_fs
+    def test_lowercase_dir_returns_snapshot_path(self, hf_cache):
+        snap = _make_cache(str(hf_cache), "models--qwen--qwen3.6-27b-fp8")
+        assert _resolve_cached_model_path(MODEL) == snap
+
+    def test_lowercase_dir_without_snapshots_returns_repo_id(self, hf_cache):
+        # Dir exists but has no snapshots subdir -> nothing to resolve to.
+        os.makedirs(os.path.join(str(hf_cache), "models--qwen--qwen3.6-27b-fp8"))
+        assert _resolve_cached_model_path(MODEL) == MODEL
+
+    @requires_case_sensitive_fs
+    def test_lowercase_dir_picks_latest_snapshot(self, hf_cache):
+        folder = "models--qwen--qwen3.6-27b-fp8"
+        _make_cache(str(hf_cache), folder, snapshot="aaaa")
+        latest = _make_cache(str(hf_cache), folder, snapshot="zzzz")
+        assert _resolve_cached_model_path(MODEL) == latest
+
+
+class TestGetEngineArgsServedName:
+    """Issue #310: served name must be decoupled from the resolved on-disk path."""
+
+    @pytest.fixture(autouse=True)
+    def base_env(self, monkeypatch):
+        # Avoid the network branch in _resolve_max_model_len.
+        monkeypatch.setenv("MAX_NUM_BATCHED_TOKENS", "2048")
+        monkeypatch.delenv("SERVED_MODEL_NAME", raising=False)
+        # Don't pick up a stray vLLM config file from the environment.
+        monkeypatch.setenv("VLLM_CONFIG_FILE", "/nonexistent-vllm-config.yaml")
+
+    @requires_case_sensitive_fs
+    def test_served_name_is_repo_id_when_path_rewritten(self, hf_cache, monkeypatch):
+        snap = _make_cache(str(hf_cache), "models--qwen--qwen3.6-27b-fp8")
+        monkeypatch.setenv("MODEL_NAME", MODEL)
+
+        result = get_engine_args()
+
+        assert result.model == snap  # weights load from the lowercase cache
+        assert result.served_model_name == MODEL  # API still serves the repo id
+
+    def test_served_name_untouched_when_no_rewrite(self, hf_cache, monkeypatch):
+        _make_cache(str(hf_cache), "models--Qwen--Qwen3.6-27B-FP8")
+        monkeypatch.setenv("MODEL_NAME", MODEL)
+
+        result = get_engine_args()
+
+        assert result.model == MODEL
+        assert result.served_model_name is None
+
+    def test_explicit_served_name_not_overridden(self, hf_cache, monkeypatch):
+        _make_cache(str(hf_cache), "models--qwen--qwen3.6-27b-fp8")
+        monkeypatch.setenv("MODEL_NAME", MODEL)
+        monkeypatch.setenv("SERVED_MODEL_NAME", "custom-name")
+
+        result = get_engine_args()
+
+        assert result.served_model_name == "custom-name"