owen/worker-vllm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update vllm version 0.5.4

Browse Source 
Signed-off-by: pandyamarut <pandyamarut@gmail.com>
This commit is contained in:
pandyamarut
2024-08-09 12:01:42 -07:00
parent f023f57217
commit 9cb9336cf5
3 changed files with 9 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/engine.py
+2 -2
View File
@@ -126,7 +126,7 @@ class OpenAIvLLMEngine(vLLMEngine):
self.model_config = await self.llm.get_model_config()
self.chat_engine = OpenAIServingChat(
engine=self.llm,
async_engine_client=self.llm,
model_config=self.model_config,
served_model_names=[self.served_model_name],
response_role=self.response_role,
@@ -136,7 +136,7 @@ class OpenAIvLLMEngine(vLLMEngine):
request_logger=None
)
self.completion_engine = OpenAIServingCompletion(
engine=self.llm,
async_engine_client=self.llm,
model_config=self.model_config,
served_model_names=[self.served_model_name],
lora_modules=[],
src/engine_args.py
+6 -6
View File
@@ -13,9 +13,9 @@ RENAME_ARGS_MAP = {
}
DEFAULT_ARGS = {
"disable_log_stats": True,
"disable_log_requests": True,
"gpu_memory_utilization": 0.9,
"disable_log_stats": os.getenv('DISABLE_LOG_STATS', 'False').lower() == 'true',
"disable_log_requests": os.getenv('DISABLE_LOG_REQUESTS', 'False').lower() == 'true',
"gpu_memory_utilization": int(os.getenv('GPU_MEMORY_UTILIZATION', 0.9)),
"pipeline_parallel_size": int(os.getenv('PIPELINE_PARALLEL_SIZE', 1)),
"tensor_parallel_size": int(os.getenv('TENSOR_PARALLEL_SIZE', 1)),
"served_model_name": os.getenv('SERVED_MODEL_NAME', None),
@@ -162,8 +162,8 @@ def get_engine_args():
args["max_seq_len_to_capture"] = int(os.getenv("MAX_CONTEXT_LEN_TO_CAPTURE"))
logging.warning("Using MAX_CONTEXT_LEN_TO_CAPTURE is deprecated. Please use MAX_SEQ_LEN_TO_CAPTURE instead.")
if "gemma-2" in args.get("model", "").lower():
os.environ["VLLM_ATTENTION_BACKEND"] = "FLASHINFER"
logging.info("Using FLASHINFER for gemma-2 model.")
# if "gemma-2" in args.get("model", "").lower():
# os.environ["VLLM_ATTENTION_BACKEND"] = "FLASHINFER"
# logging.info("Using FLASHINFER for gemma-2 model.")
return AsyncEngineArgs(**args)