owen/worker-vllm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #133 from kldzj/tool-calling

Browse Source 
feat: add tool calling flags
This commit is contained in:
Marut Pandya
2024-11-29 16:29:56 -08:00
committed by GitHub
parent 0a89394f1d 8df7f41f1d
commit 149da95cd0
2 changed files with 31 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/engine_args.py
+3 -1
View File
@@ -89,7 +89,9 @@ DEFAULT_ARGS = {
"qlora_adapter_name_or_path": os.getenv('QLORA_ADAPTER_NAME_OR_PATH', None), "qlora_adapter_name_or_path": os.getenv('QLORA_ADAPTER_NAME_OR_PATH', None),
"disable_logprobs_during_spec_decoding": os.getenv('DISABLE_LOGPROBS_DURING_SPEC_DECODING', None), "disable_logprobs_during_spec_decoding": os.getenv('DISABLE_LOGPROBS_DURING_SPEC_DECODING', None),
"otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None), "otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None),
"use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true') "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true'),
"enable_auto_tool_choice": os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true',
"tool_call_parser": os.getenv('TOOL_CALL_PARSER', "") or None
} }
def match_vllm_args(args): def match_vllm_args(args):
worker-config.json
+28 -2
View File
@@ -24,7 +24,8 @@
"NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
"TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
"MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
"PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST" "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
"ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
] ]
}, },
{ {
@@ -84,7 +85,8 @@
"NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
"TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
"MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
"PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST" "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
"ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
] ]
}, },
{ {
@@ -991,6 +993,30 @@
"description": "Enables or disables vLLM request logging", "description": "Enables or disables vLLM request logging",
"required": false, "required": false,
"type": "toggle" "type": "toggle"
},
"ENABLE_AUTO_TOOL_CHOICE": {
"env_var_name": "ENABLE_AUTO_TOOL_CHOICE",
"value": false,
"title": "Enable Auto Tool Choice",
"description": "Enables or disables auto tool choice",
"required": false,
"type": "toggle"
},
"TOOL_CALL_PARSER": {
"env_var_name": "TOOL_CALL_PARSER",
"value": "",
"title": "Tool Call Parser",
"description": "Tool call parser",
"required": false,
"type": "select",
"options": [
{ "value": "", "label": "None" },
{ "value": "hermes", "label": "Hermes" },
{ "value": "mistral", "label": "Mistral" },
{ "value": "llama3_json", "label": "Llama3 JSON" },
{ "value": "pythonic", "label": "Pythonic" },
{ "value": "internlm", "label": "InternLM" }
]
} }
} }
} }