fix oai completion api error

Signed-off-by: pandyamarut <pandyamarut@gmail.com>
This commit is contained in:
pandyamarut
2024-09-06 11:03:47 -07:00
parent f3a530b7fe
commit 814f50af38
2 changed files with 5 additions and 2 deletions
+2 -1
View File
@@ -176,7 +176,8 @@ class OpenAIvLLMEngine(vLLMEngine):
yield create_error_response(str(e)).model_dump()
return
response_generator = await generator_function(request, raw_request=None)
dummy_request = DummyRequest()
response_generator = await generator_function(request, raw_request=dummy_request)
if not openai_request.openai_input.get("stream") or isinstance(response_generator, ErrorResponse):
yield response_generator.model_dump()
+2
View File
@@ -49,6 +49,8 @@ class JobInput:
self.openai_input = job.get("openai_input")
class DummyRequest:
def __init__(self):
self.headers = {}
async def is_disconnected(self):
return False