chore(deps): update runpod to latest version (#242)

* enable expert parallel arg for moe models

* add ENABLE_EXPERT_PARALLEL to hub config

.runpod/hub.json

@@ -6,20 +6,10 @@
   "iconUrl": "https://registry.npmmirror.com/@lobehub/icons-static-png/latest/files/dark/vllm-color.png",
   "config": {
     "runsOn": "GPU",
-    "containerDiskInGb": 200,
+    "containerDiskInGb": 150,
-    "gpuIds": "ADA_80_PRO, AMPERE_80",
+    "gpuIds": "ADA_80_PRO,AMPERE_80",
     "gpuCount": 1,
-    "allowedCudaVersions": [
+    "allowedCudaVersions": ["12.9", "12.8", "12.7", "12.6", "12.5", "12.4"],
-      "12.9",
-      "12.8",
-      "12.7",
-      "12.6",
-      "12.5",
-      "12.4",
-      "12.3",
-      "12.2",
-      "12.1"
-    ],
     "presets": [
       {
         "name": "deepseek-ai/deepseek-r1-distill-llama-8b",
@@ -935,7 +925,17 @@
           "name": "Max Concurrency",
           "type": "number",
           "description": "Max concurrent requests per worker. vLLM has an internal queue, so you don't have to worry about limiting by VRAM, this is for improving scaling/load balancing efficiency",
-          "default": 300,
+          "default": 30,
+          "advanced": true
+        }
+      },
+      {
+        "key": "ENABLE_EXPERT_PARALLEL",
+        "input": {
+          "name": "Enable Expert Parallel",
+          "type": "boolean",
+          "description": "Enable Expert Parallel for MoE models",
+          "default": false,
           "advanced": true
         }
       },

.runpod/tests.json

@@ -38,14 +38,6 @@
         "value": "HuggingFaceTB/SmolLM2-135M-Instruct"
       }
     ],
-    "allowedCudaVersions": [
+    "allowedCudaVersions": ["12.9", "12.8", "12.7", "12.6", "12.5"]
-      "12.7",
-      "12.6",
-      "12.5",
-      "12.4",
-      "12.3",
-      "12.2",
-      "12.1"
-    ]
   }
 }

Dockerfile

@@ -12,7 +12,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     python3 -m pip install --upgrade -r /requirements.txt
 
 # Install vLLM (switching back to pip installs since issues that required building fork are fixed and space optimization is not as important since caching) and FlashInfer 
-RUN python3 -m pip install vllm==0.10.0 && \
+RUN python3 -m pip install vllm==0.11.0 && \
     python3 -m pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3
 
 # Setup for Option 2: Building the Image with the Model included

README.md

@@ -57,7 +57,7 @@ Configure worker-vllm using environment variables:
 | `ENABLE_AUTO_TOOL_CHOICE`           | Enable automatic tool selection                   | false               | boolean (true or false)                                            |
 | `TOOL_CALL_PARSER`                  | Parser for tool calls                             |                     | "mistral", "hermes", "llama3_json", "granite", "deepseek_v3", etc. |
 | `OPENAI_SERVED_MODEL_NAME_OVERRIDE` | Override served model name in API                 |                     | String                                                             |
-| `MAX_CONCURRENCY`                   | Maximum concurrent requests                       | 300                 | Integer                                                            |
+| `MAX_CONCURRENCY`                   | Maximum concurrent requests                       | 30                  | Integer                                                            |
 
 For the complete list of all available environment variables, examples, and detailed descriptions: **[Configuration](docs/configuration.md)**
 

builder/requirements.txt

@@ -1,14 +1,14 @@
 ray
 pandas
 pyarrow
-runpod~=1.7.7
+runpod>=1.8,<2.0
 huggingface-hub
 packaging
 typing-extensions>=4.8.0
 pydantic
 pydantic-settings
 hf-transfer
-transformers>=4.55.0
+transformers>=4.57.0
 bitsandbytes>=0.45.0
 kernels
 torch==2.6.0

docs/configuration.md

@@ -85,6 +85,7 @@ Complete guide to all environment variables and configuration options for worker
 | `ENFORCE_EAGER`                | False   | `bool`          | Always use eager-mode PyTorch. If False(`0`), will use eager mode and CUDA graph in hybrid for maximal performance and flexibility. |
 | `MAX_SEQ_LEN_TO_CAPTURE`       | `8192`  | `int`           | Maximum context length covered by CUDA graphs. When a sequence has context length larger than this, we fall back to eager mode.     |
 | `DISABLE_CUSTOM_ALL_REDUCE`    | `0`     | `int`           | Enables or disables custom all reduce.                                                                                              |
+| `ENABLE_EXPERT_PARALLEL`       | `False` | `bool`           |  Enable Expert Parallel for MoE models  |
 
 ## Tokenizer Settings
 
@@ -119,7 +120,7 @@ The way this works is that the first request will have a batch size of `DEFAULT_
 
 | Variable               | Default | Type/Choices | Description                                                                                                                                                                |
 | ---------------------- | ------- | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `MAX_CONCURRENCY`      | `300`   | `int`        | Max concurrent requests per worker. vLLM has an internal queue, so you don't have to worry about limiting by VRAM, this is for improving scaling/load balancing efficiency |
+| `MAX_CONCURRENCY`      | `30`    | `int`        | Max concurrent requests per worker. vLLM has an internal queue, so you don't have to worry about limiting by VRAM, this is for improving scaling/load balancing efficiency |
 | `DISABLE_LOG_STATS`    | False   | `bool`       | Enables or disables vLLM stats logging.                                                                                                                                    |
 | `DISABLE_LOG_REQUESTS` | False   | `bool`       | Enables or disables vLLM request logging.                                                                                                                                  |
 

docs/conventions.md

@@ -51,7 +51,8 @@ RunPod Request → handler.py → JobInput → Engine Selection → vLLM Generat
 
 - `src/engine_args.py`: Centralized configuration management
 - `src/constants.py`: Default values for core settings
-- `worker-config.json`: UI form generation for RunPod console
+- `.runpod/hub.json`: Hub UI configuration (CRITICAL: always update when changing defaults)
+- `worker-config.json`: UI form generation for RunPod console (if exists)
 
 ## Core Development Concepts
 
@@ -222,7 +223,7 @@ src/
 
 ### 2. **Concurrency Patterns**
 
-- **Max Concurrency**: 300 concurrent requests by default
+- **Max Concurrency**: 30 concurrent requests by default
 - **vLLM Queuing**: Internal request batching and scheduling
 - **RunPod Integration**: Concurrency modifier for auto-scaling
 

src/constants.py

@@ -1,4 +1,4 @@
 DEFAULT_BATCH_SIZE = 50
-DEFAULT_MAX_CONCURRENCY = 300
+DEFAULT_MAX_CONCURRENCY = 30
 DEFAULT_BATCH_SIZE_GROWTH_FACTOR = 3
 DEFAULT_MIN_BATCH_SIZE = 1

src/engine_args.py

@@ -80,6 +80,7 @@ DEFAULT_ARGS = {
     "guided_decoding_backend": os.getenv('GUIDED_DECODING_BACKEND', 'outlines'),
     "speculative_model": os.getenv('SPECULATIVE_MODEL', None),
     "speculative_draft_tensor_parallel_size": int(os.getenv('SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE', 0)) or None,
+    "enable_expert_parallel": bool(os.getenv('ENABLE_EXPERT_PARALLEL', 'False').lower() == 'true'),
     "num_speculative_tokens": int(os.getenv('NUM_SPECULATIVE_TOKENS', 0)) or None,
     "speculative_max_model_len": int(os.getenv('SPECULATIVE_MAX_MODEL_LEN', 0)) or None,
     "speculative_disable_by_batch_size": int(os.getenv('SPECULATIVE_DISABLE_BY_BATCH_SIZE', 0)) or None,