From afa33a2875df4394465dab744985b6dbee1eb54f Mon Sep 17 00:00:00 2001 From: alpayariyak Date: Thu, 1 Feb 2024 03:01:51 +0000 Subject: [PATCH] Handle errors --- src/engine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/engine.py b/src/engine.py index f80f3c9..b518d7a 100644 --- a/src/engine.py +++ b/src/engine.py @@ -5,7 +5,7 @@ import json from torch.cuda import device_count from vllm import AsyncLLMEngine, AsyncEngineArgs, SamplingParams from vllm.entrypoints.openai.serving_chat import OpenAIServingChat -from vllm.entrypoints.openai.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.protocol import ChatCompletionRequest, ErrorResponse from transformers import AutoTokenizer from utils import count_physical_cores, DummyRequest from constants import DEFAULT_MAX_CONCURRENCY @@ -124,6 +124,8 @@ class vLLMEngine: ) response_generator = await self.openai_engine.create_chat_completion(chat_completion_request, DummyRequest()) + if isinstance(response_generator, ErrorResponse): + raise ValueError(response_generator.model_dump()) if not stream: yield response_generator else: