fix metrics

This commit is contained in:
Jorg Doku
2023-08-29 22:17:50 -05:00
parent 5fd458381d
commit 476498b9bc
+1 -2
View File
@@ -43,7 +43,7 @@ engine_args = AsyncEngineArgs(
llm = AsyncLLMEngine.from_engine_args(engine_args)
# Incorporate metrics tracking
llm.engine._log_system_stats = vllm_log_system_stats
llm.engine._log_system_stats = lambda x, y: vllm_log_system_stats(llm.engine, x, y)
def concurrency_controller() -> bool:
# Compute pending sequences
@@ -114,7 +114,6 @@ def validate_sampling_params(sampling_params):
'logprobs': logprobs,
}
async def handler_streaming(job: dict) -> Generator[dict[str, list], None, None]:
'''
This is the handler function that will be called by the serverless worker.