diff --git a/src/engine_args.py b/src/engine_args.py index 45e50d1..485e505 100644 --- a/src/engine_args.py +++ b/src/engine_args.py @@ -89,7 +89,9 @@ DEFAULT_ARGS = { "qlora_adapter_name_or_path": os.getenv('QLORA_ADAPTER_NAME_OR_PATH', None), "disable_logprobs_during_spec_decoding": os.getenv('DISABLE_LOGPROBS_DURING_SPEC_DECODING', None), "otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None), - "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true') + "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true'), + "enable_auto_tool_choice": os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true', + "tool_call_parser": os.getenv('TOOL_CALL_PARSER', None) } def match_vllm_args(args): diff --git a/worker-config.json b/worker-config.json index 6b39e3e..2c077ed 100644 --- a/worker-config.json +++ b/worker-config.json @@ -24,7 +24,8 @@ "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST" + "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", + "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" ] }, { @@ -84,7 +85,8 @@ "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST" + "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", + "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" ] }, { @@ -991,6 +993,30 @@ "description": "Enables or disables vLLM request logging", "required": false, "type": "toggle" + }, + "ENABLE_AUTO_TOOL_CHOICE": { + "env_var_name": "ENABLE_AUTO_TOOL_CHOICE", + "value": false, + "title": "Enable Auto Tool Choice", + "description": "Enables or disables auto tool choice", + "required": false, + "type": "toggle" + }, + "TOOL_CALL_PARSER": { + "env_var_name": "TOOL_CALL_PARSER", + "value": "", + "title": "Tool Call Parser", + "description": "Tool call parser", + "required": false, + "type": "select", + "options": [ + { "value": "", "label": "None" }, + { "value": "hermes", "label": "Hermes" }, + { "value": "mistral", "label": "Mistral" }, + { "value": "llama3_json", "label": "Llama3 JSON" }, + { "value": "pythonic", "label": "Pythonic" }, + { "value": "internlm", "label": "InternLM" } + ] } } } \ No newline at end of file