Fix: Yield if tokens left in batch

Temp: default model for testing image
2024-02-01 00:52:39 +00:00
parent dab8bad906
commit 3bbcf0021b
2 changed files with 7 additions and 1 deletions
@@ -138,7 +138,8 @@ class vLLMEngine:
                        yield batch
                        batch = ""
                        batch_token_counter = 0
-                        
+            if batch:
+                yield batch
               
    def _initialize_config(self):
        quantization = self._get_quantization()