From b0e7b575f394d2e949c859aed4ef9f89c89ce1fa Mon Sep 17 00:00:00 2001
From: Samuel Will <will.samuel@protonmail.com>
Date: Fri, 9 Feb 2024 10:24:58 +0000
Subject: [PATCH] fix: default value for tokenizer

---
 builder/download_model.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/builder/download_model.py b/builder/download_model.py
index caf4a42..4d5ea41 100644
--- a/builder/download_model.py
+++ b/builder/download_model.py
@@ -28,6 +28,8 @@ def move_files(src_dir, dest_dir):
 if __name__ == "__main__":
     model, download_dir = os.getenv("MODEL_NAME"), os.getenv("HF_HOME")
     tokenizer = os.getenv("TOKENIZER_NAME", model)
+    if len(tokenizer) == 0:
+        tokenizer = model
     revisions = {
         "model": os.getenv("MODEL_REVISION") or None,
         "tokenizer": os.getenv("TOKENIZER_REVISION") or None