From 4f5e0d37c49a60156ac4912aea2e55090c46ed00 Mon Sep 17 00:00:00 2001 From: alpayariyak Date: Thu, 8 Feb 2024 23:53:13 +0000 Subject: [PATCH 1/4] Fix tokenizer's trust_remote_code parameter --- src/engine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/engine.py b/src/engine.py index b165943..f4f0789 100644 --- a/src/engine.py +++ b/src/engine.py @@ -13,8 +13,8 @@ from dotenv import load_dotenv class Tokenizer: - def __init__(self, tokenizer_name_or_path, tokenizer_revision): - self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, revision=tokenizer_revision) + def __init__(self, tokenizer_name_or_path, tokenizer_revision, trust_remote_code): + self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, revision=tokenizer_revision, trust_remote_code=trust_remote_code) self.custom_chat_template = os.getenv("CUSTOM_CHAT_TEMPLATE") self.has_chat_template = bool(self.tokenizer.chat_template) or bool(self.custom_chat_template) if self.custom_chat_template and isinstance(self.custom_chat_template, str): @@ -41,7 +41,7 @@ class vLLMEngine: load_dotenv() # For local development self.config = self._initialize_config() logging.info("vLLM config: %s", self.config) - self.tokenizer = Tokenizer(self.config["tokenizer"], self.config["tokenizer_revision"]) + self.tokenizer = Tokenizer(self.config["tokenizer"], self.config["tokenizer_revision"], self.config["trust_remote_code"]) self.llm = self._initialize_llm() if engine is None else engine self.openai_engine = self._initialize_openai() self.max_concurrency = int(os.getenv("MAX_CONCURRENCY", DEFAULT_MAX_CONCURRENCY)) From b0e7b575f394d2e949c859aed4ef9f89c89ce1fa Mon Sep 17 00:00:00 2001 From: Samuel Will Date: Fri, 9 Feb 2024 10:24:58 +0000 Subject: [PATCH 2/4] fix: default value for tokenizer --- builder/download_model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builder/download_model.py b/builder/download_model.py index caf4a42..4d5ea41 100644 --- a/builder/download_model.py +++ b/builder/download_model.py @@ -28,6 +28,8 @@ def move_files(src_dir, dest_dir): if __name__ == "__main__": model, download_dir = os.getenv("MODEL_NAME"), os.getenv("HF_HOME") tokenizer = os.getenv("TOKENIZER_NAME", model) + if len(tokenizer) == 0: + tokenizer = model revisions = { "model": os.getenv("MODEL_REVISION") or None, "tokenizer": os.getenv("TOKENIZER_REVISION") or None From 7b3fd05542ea47fd1b37c8cd7b65319fd5ae50ad Mon Sep 17 00:00:00 2001 From: alpayariyak Date: Fri, 9 Feb 2024 22:49:17 -0500 Subject: [PATCH 3/4] Small refactor to tokenizer fix --- builder/download_model.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/builder/download_model.py b/builder/download_model.py index 4d5ea41..dddcf56 100644 --- a/builder/download_model.py +++ b/builder/download_model.py @@ -27,9 +27,8 @@ def move_files(src_dir, dest_dir): if __name__ == "__main__": model, download_dir = os.getenv("MODEL_NAME"), os.getenv("HF_HOME") - tokenizer = os.getenv("TOKENIZER_NAME", model) - if len(tokenizer) == 0: - tokenizer = model + tokenizer = os.getenv("TOKENIZER_NAME") or model + revisions = { "model": os.getenv("MODEL_REVISION") or None, "tokenizer": os.getenv("TOKENIZER_REVISION") or None From 2941db0fb848b8b2f57249b2ae5f3e2dd45a36fd Mon Sep 17 00:00:00 2001 From: Alpay Ariyak <98838263+alpayariyak@users.noreply.github.com> Date: Fri, 9 Feb 2024 23:12:29 -0500 Subject: [PATCH 4/4] Update worker-vllm version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2fb7bad..eee79ee 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ We now offer a pre-built Docker Image for the vLLM Worker that you can configure
-Stable Image: ```runpod/worker-vllm:0.2.2``` +Stable Image: ```runpod/worker-vllm:0.2.3``` Development Image: ```runpod/worker-vllm:dev```