fix metrics
This commit is contained in:
+1
-2
@@ -43,7 +43,7 @@ engine_args = AsyncEngineArgs(
|
||||
llm = AsyncLLMEngine.from_engine_args(engine_args)
|
||||
|
||||
# Incorporate metrics tracking
|
||||
llm.engine._log_system_stats = vllm_log_system_stats
|
||||
llm.engine._log_system_stats = lambda x, y: vllm_log_system_stats(llm.engine, x, y)
|
||||
|
||||
def concurrency_controller() -> bool:
|
||||
# Compute pending sequences
|
||||
@@ -114,7 +114,6 @@ def validate_sampling_params(sampling_params):
|
||||
'logprobs': logprobs,
|
||||
}
|
||||
|
||||
|
||||
async def handler_streaming(job: dict) -> Generator[dict[str, list], None, None]:
|
||||
'''
|
||||
This is the handler function that will be called by the serverless worker.
|
||||
|
||||
Reference in New Issue
Block a user