update

2023-08-22 15:40:09 -05:00
parent 177c8c57c0
commit 6498c67340
1 changed files with 2 additions and 1 deletions
@@ -36,7 +36,8 @@ engine_args = AsyncEngineArgs(
    dtype="auto",
    seed=0,
    worker_use_ray=False,
-    max_num_batched_tokens=10000
+    max_num_batched_tokens=10000,
+    max_num_seqs=1000
 )

 # Create the vLLM asynchronous engine