owen/worker-vllm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update vllm version 0.5.4

Browse Source 
Signed-off-by: pandyamarut <pandyamarut@gmail.com>
This commit is contained in:
pandyamarut
2024-08-09 12:01:42 -07:00
parent f023f57217
commit 9cb9336cf5
3 changed files with 9 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+1 -1
View File
@@ -12,7 +12,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
python3 -m pip install --upgrade -r /requirements.txt python3 -m pip install --upgrade -r /requirements.txt
# Install vLLM (switching back to pip installs since issues that required building fork are fixed and space optimization is not as important since caching) and FlashInfer # Install vLLM (switching back to pip installs since issues that required building fork are fixed and space optimization is not as important since caching) and FlashInfer
RUN python3 -m pip install vllm==0.5.3.post1 && \ RUN python3 -m pip install vllm==0.5.4 && \
python3 -m pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3 python3 -m pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3
# Setup for Option 2: Building the Image with the Model included # Setup for Option 2: Building the Image with the Model included
src/engine.py
+2 -2
View File
@@ -126,7 +126,7 @@ class OpenAIvLLMEngine(vLLMEngine):
self.model_config = await self.llm.get_model_config() self.model_config = await self.llm.get_model_config()
self.chat_engine = OpenAIServingChat( self.chat_engine = OpenAIServingChat(
engine=self.llm, async_engine_client=self.llm,
model_config=self.model_config, model_config=self.model_config,
served_model_names=[self.served_model_name], served_model_names=[self.served_model_name],
response_role=self.response_role, response_role=self.response_role,
@@ -136,7 +136,7 @@ class OpenAIvLLMEngine(vLLMEngine):
request_logger=None request_logger=None
) )
self.completion_engine = OpenAIServingCompletion( self.completion_engine = OpenAIServingCompletion(
engine=self.llm, async_engine_client=self.llm,
model_config=self.model_config, model_config=self.model_config,
served_model_names=[self.served_model_name], served_model_names=[self.served_model_name],
lora_modules=[], lora_modules=[],
src/engine_args.py
+6 -6
View File
@@ -13,9 +13,9 @@ RENAME_ARGS_MAP = {
} }
DEFAULT_ARGS = { DEFAULT_ARGS = {
"disable_log_stats": True, "disable_log_stats": os.getenv('DISABLE_LOG_STATS', 'False').lower() == 'true',
"disable_log_requests": True, "disable_log_requests": os.getenv('DISABLE_LOG_REQUESTS', 'False').lower() == 'true',
"gpu_memory_utilization": 0.9, "gpu_memory_utilization": int(os.getenv('GPU_MEMORY_UTILIZATION', 0.9)),
"pipeline_parallel_size": int(os.getenv('PIPELINE_PARALLEL_SIZE', 1)), "pipeline_parallel_size": int(os.getenv('PIPELINE_PARALLEL_SIZE', 1)),
"tensor_parallel_size": int(os.getenv('TENSOR_PARALLEL_SIZE', 1)), "tensor_parallel_size": int(os.getenv('TENSOR_PARALLEL_SIZE', 1)),
"served_model_name": os.getenv('SERVED_MODEL_NAME', None), "served_model_name": os.getenv('SERVED_MODEL_NAME', None),
@@ -162,8 +162,8 @@ def get_engine_args():
args["max_seq_len_to_capture"] = int(os.getenv("MAX_CONTEXT_LEN_TO_CAPTURE")) args["max_seq_len_to_capture"] = int(os.getenv("MAX_CONTEXT_LEN_TO_CAPTURE"))
logging.warning("Using MAX_CONTEXT_LEN_TO_CAPTURE is deprecated. Please use MAX_SEQ_LEN_TO_CAPTURE instead.") logging.warning("Using MAX_CONTEXT_LEN_TO_CAPTURE is deprecated. Please use MAX_SEQ_LEN_TO_CAPTURE instead.")
if "gemma-2" in args.get("model", "").lower(): # if "gemma-2" in args.get("model", "").lower():
os.environ["VLLM_ATTENTION_BACKEND"] = "FLASHINFER" # os.environ["VLLM_ATTENTION_BACKEND"] = "FLASHINFER"
logging.info("Using FLASHINFER for gemma-2 model.") # logging.info("Using FLASHINFER for gemma-2 model.")
return AsyncEngineArgs(**args) return AsyncEngineArgs(**args)