Make handler stream aggregate_text

This commit is contained in:
alpayariyak
2023-12-01 04:21:03 +00:00
parent 8dbf197362
commit daf2f4d07b
+3 -1
View File
@@ -130,13 +130,15 @@ async def handler_streaming(job: dict) -> Generator[dict[str, list], None, None]
request_id = utils.random_uuid()
results_generator = llm.generate(prompt, sampling_params, request_id)
aggregate_text = ""
last_output_text = ""
async for request_output in results_generator:
for output in request_output.outputs:
if output.text:
yield {"text": output.text[len(last_output_text):]}
last_output_text = output.text
aggregate_text += output.text
yield {"aggregate_text": aggregate_text}
runpod.serverless.start({