diff --git a/src/handler.py b/src/handler.py index 2352c87..610ce16 100644 --- a/src/handler.py +++ b/src/handler.py @@ -43,7 +43,7 @@ engine_args = AsyncEngineArgs( llm = AsyncLLMEngine.from_engine_args(engine_args) # Incorporate metrics tracking -llm.engine._log_system_stats = vllm_log_system_stats +llm.engine._log_system_stats = lambda x, y: vllm_log_system_stats(llm.engine, x, y) def concurrency_controller() -> bool: # Compute pending sequences @@ -114,7 +114,6 @@ def validate_sampling_params(sampling_params): 'logprobs': logprobs, } - async def handler_streaming(job: dict) -> Generator[dict[str, list], None, None]: ''' This is the handler function that will be called by the serverless worker.