From 6498c67340c2dc659597a7105c1055916e59e5fd Mon Sep 17 00:00:00 2001
From: Jorg Doku <jd728@cornell.edu>
Date: Tue, 22 Aug 2023 15:40:09 -0500
Subject: [PATCH] update

---
 src/handler.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/handler.py b/src/handler.py
index 67b119e..5892a4b 100644
--- a/src/handler.py
+++ b/src/handler.py
@@ -36,7 +36,8 @@ engine_args = AsyncEngineArgs(
     dtype="auto",
     seed=0,
     worker_use_ray=False,
-    max_num_batched_tokens=10000
+    max_num_batched_tokens=10000,
+    max_num_seqs=1000
 )
 
 # Create the vLLM asynchronous engine