owen/worker-vllm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #143 from sihamouda/main

Browse Source 
Add limit_mm_per_prompt engine argument for multimodels input
This commit is contained in:
Marut Pandya
2025-01-27 16:41:47 -08:00
committed by GitHub
parent a578c6df23 0e3359a70e
commit 610429df23
2 changed files with 7 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/engine_args.py
+3 -1
View File
@@ -4,6 +4,7 @@ import logging
from torch.cuda import device_count from torch.cuda import device_count
from vllm import AsyncEngineArgs from vllm import AsyncEngineArgs
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
from src.utils import convert_limit_mm_per_prompt
RENAME_ARGS_MAP = { RENAME_ARGS_MAP = {
"MODEL_NAME": "model", "MODEL_NAME": "model",
@@ -89,7 +90,8 @@ DEFAULT_ARGS = {
"qlora_adapter_name_or_path": os.getenv('QLORA_ADAPTER_NAME_OR_PATH', None), "qlora_adapter_name_or_path": os.getenv('QLORA_ADAPTER_NAME_OR_PATH', None),
"disable_logprobs_during_spec_decoding": os.getenv('DISABLE_LOGPROBS_DURING_SPEC_DECODING', None), "disable_logprobs_during_spec_decoding": os.getenv('DISABLE_LOGPROBS_DURING_SPEC_DECODING', None),
"otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None), "otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None),
"use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true') "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true'),
"limit_mm_per_prompt": convert_limit_mm_per_prompt(os.getenv('LIMIT_MM_PER_PROMPT', "image=1"))
} }
def match_vllm_args(args): def match_vllm_args(args):
src/utils.py
+4
View File
@@ -15,6 +15,10 @@ except ImportError:
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
def convert_limit_mm_per_prompt(input_string: str):
key, value = input_string.split('=')
return {key: int(value)}
def count_physical_cores(): def count_physical_cores():
with open('/proc/cpuinfo') as f: with open('/proc/cpuinfo') as f:
content = f.readlines() content = f.readlines()