From 4d7b8c03c0ef17ca345b74beea531059fbe5e682 Mon Sep 17 00:00:00 2001 From: Nikolai Kolodziej Date: Sun, 24 Nov 2024 06:51:41 +0100 Subject: [PATCH 1/2] feat: tool calling flags --- src/engine_args.py | 4 +++- worker-config.json | 30 ++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/engine_args.py b/src/engine_args.py index 45e50d1..485e505 100644 --- a/src/engine_args.py +++ b/src/engine_args.py @@ -89,7 +89,9 @@ DEFAULT_ARGS = { "qlora_adapter_name_or_path": os.getenv('QLORA_ADAPTER_NAME_OR_PATH', None), "disable_logprobs_during_spec_decoding": os.getenv('DISABLE_LOGPROBS_DURING_SPEC_DECODING', None), "otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None), - "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true') + "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true'), + "enable_auto_tool_choice": os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true', + "tool_call_parser": os.getenv('TOOL_CALL_PARSER', None) } def match_vllm_args(args): diff --git a/worker-config.json b/worker-config.json index 6b39e3e..2c077ed 100644 --- a/worker-config.json +++ b/worker-config.json @@ -24,7 +24,8 @@ "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST" + "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", + "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" ] }, { @@ -84,7 +85,8 @@ "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST" + "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", + "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" ] }, { @@ -991,6 +993,30 @@ "description": "Enables or disables vLLM request logging", "required": false, "type": "toggle" + }, + "ENABLE_AUTO_TOOL_CHOICE": { + "env_var_name": "ENABLE_AUTO_TOOL_CHOICE", + "value": false, + "title": "Enable Auto Tool Choice", + "description": "Enables or disables auto tool choice", + "required": false, + "type": "toggle" + }, + "TOOL_CALL_PARSER": { + "env_var_name": "TOOL_CALL_PARSER", + "value": "", + "title": "Tool Call Parser", + "description": "Tool call parser", + "required": false, + "type": "select", + "options": [ + { "value": "", "label": "None" }, + { "value": "hermes", "label": "Hermes" }, + { "value": "mistral", "label": "Mistral" }, + { "value": "llama3_json", "label": "Llama3 JSON" }, + { "value": "pythonic", "label": "Pythonic" }, + { "value": "internlm", "label": "InternLM" } + ] } } } \ No newline at end of file From 8df7f41f1d2392ca19216832999716b2062a0cc0 Mon Sep 17 00:00:00 2001 From: Nikolai Kolodziej Date: Sun, 24 Nov 2024 06:55:43 +0100 Subject: [PATCH 2/2] fix: set empty `tool_call_parser` to None --- src/engine_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine_args.py b/src/engine_args.py index 485e505..4c445df 100644 --- a/src/engine_args.py +++ b/src/engine_args.py @@ -91,7 +91,7 @@ DEFAULT_ARGS = { "otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None), "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true'), "enable_auto_tool_choice": os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true', - "tool_call_parser": os.getenv('TOOL_CALL_PARSER', None) + "tool_call_parser": os.getenv('TOOL_CALL_PARSER', "") or None } def match_vllm_args(args):