owen/worker-vllm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: tool calling flags

Browse Source
This commit is contained in:
Nikolai Kolodziej
2024-11-24 06:51:41 +01:00
parent 2df915a145
commit 4d7b8c03c0
2 changed files with 31 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/engine_args.py
+3 -1
View File
@@ -89,7 +89,9 @@ DEFAULT_ARGS = {
"qlora_adapter_name_or_path": os.getenv('QLORA_ADAPTER_NAME_OR_PATH', None),
"disable_logprobs_during_spec_decoding": os.getenv('DISABLE_LOGPROBS_DURING_SPEC_DECODING', None),
"otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None),
"use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true')
"use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true'),
"enable_auto_tool_choice": os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
"tool_call_parser": os.getenv('TOOL_CALL_PARSER', None)
}
def match_vllm_args(args):
worker-config.json
+28 -2
View File
@@ -24,7 +24,8 @@
"NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
"TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
"MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
"PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST"
"PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
"ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
]
},
{
@@ -84,7 +85,8 @@
"NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
"TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
"MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
"PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST"
"PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
"ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
]
},
{
@@ -991,6 +993,30 @@
"description": "Enables or disables vLLM request logging",
"required": false,
"type": "toggle"
},
"ENABLE_AUTO_TOOL_CHOICE": {
"env_var_name": "ENABLE_AUTO_TOOL_CHOICE",
"value": false,
"title": "Enable Auto Tool Choice",
"description": "Enables or disables auto tool choice",
"required": false,
"type": "toggle"
},
"TOOL_CALL_PARSER": {
"env_var_name": "TOOL_CALL_PARSER",
"value": "",
"title": "Tool Call Parser",
"description": "Tool call parser",
"required": false,
"type": "select",
"options": [
{ "value": "", "label": "None" },
{ "value": "hermes", "label": "Hermes" },
{ "value": "mistral", "label": "Mistral" },
{ "value": "llama3_json", "label": "Llama3 JSON" },
{ "value": "pythonic", "label": "Pythonic" },
{ "value": "internlm", "label": "InternLM" }
]
}
}
}