From 7b3fd05542ea47fd1b37c8cd7b65319fd5ae50ad Mon Sep 17 00:00:00 2001 From: alpayariyak Date: Fri, 9 Feb 2024 22:49:17 -0500 Subject: [PATCH] Small refactor to tokenizer fix --- builder/download_model.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/builder/download_model.py b/builder/download_model.py index 4d5ea41..dddcf56 100644 --- a/builder/download_model.py +++ b/builder/download_model.py @@ -27,9 +27,8 @@ def move_files(src_dir, dest_dir): if __name__ == "__main__": model, download_dir = os.getenv("MODEL_NAME"), os.getenv("HF_HOME") - tokenizer = os.getenv("TOKENIZER_NAME", model) - if len(tokenizer) == 0: - tokenizer = model + tokenizer = os.getenv("TOKENIZER_NAME") or model + revisions = { "model": os.getenv("MODEL_REVISION") or None, "tokenizer": os.getenv("TOKENIZER_REVISION") or None