Fix Concurrency

2023-12-28 22:56:44 +00:00
parent fe6e618692
commit f7ac60f802
2 changed files with 6 additions and 3 deletions
@@ -10,8 +10,7 @@ vllm_engine = VLLMEngine()


 def concurrency_modifier(current_concurrency) -> int:
-    return max(0, serverless_config.max_concurrency - current_concurrency)
-
+    return max(0, serverless_config.max_concurrency - vllm_engine.get_n_current_jobs)

 async def handler(job: dict) -> Generator[dict, None, None]:
    job_input = job["input"]