From 3e86d168920303ea069189fe4caaaa0361d94676 Mon Sep 17 00:00:00 2001 From: kldzj Date: Fri, 6 Dec 2024 13:08:48 +0100 Subject: [PATCH] fix: openai tool calling --- src/engine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/engine.py b/src/engine.py index 48e8aac..e53a193 100644 --- a/src/engine.py +++ b/src/engine.py @@ -17,7 +17,7 @@ from vllm.entrypoints.openai.serving_engine import BaseModelPath, LoRAModulePath from utils import DummyRequest, JobInput, BatchSize, create_error_response from constants import DEFAULT_MAX_CONCURRENCY, DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE_GROWTH_FACTOR, DEFAULT_MIN_BATCH_SIZE from tokenizer import TokenizerWrapper -from engine_args import get_engine_args +from engine_args import get_engine_args, DEFAULT_ARGS class vLLMEngine: def __init__(self, engine = None): @@ -146,6 +146,8 @@ class OpenAIvLLMEngine(vLLMEngine): base_model_paths=self.base_model_paths, response_role=self.response_role, chat_template=self.tokenizer.tokenizer.chat_template, + enable_auto_tools=DEFAULT_ARGS.enable_auto_tool_choice, + tool_parser=DEFAULT_ARGS.tool_call_parser, lora_modules=lora_modules, prompt_adapters=None, request_logger=None