From 4d7b8c03c0ef17ca345b74beea531059fbe5e682 Mon Sep 17 00:00:00 2001 From: Nikolai Kolodziej Date: Sun, 24 Nov 2024 06:51:41 +0100 Subject: [PATCH 01/10] feat: tool calling flags --- src/engine_args.py | 4 +++- worker-config.json | 30 ++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/engine_args.py b/src/engine_args.py index 45e50d1..485e505 100644 --- a/src/engine_args.py +++ b/src/engine_args.py @@ -89,7 +89,9 @@ DEFAULT_ARGS = { "qlora_adapter_name_or_path": os.getenv('QLORA_ADAPTER_NAME_OR_PATH', None), "disable_logprobs_during_spec_decoding": os.getenv('DISABLE_LOGPROBS_DURING_SPEC_DECODING', None), "otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None), - "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true') + "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true'), + "enable_auto_tool_choice": os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true', + "tool_call_parser": os.getenv('TOOL_CALL_PARSER', None) } def match_vllm_args(args): diff --git a/worker-config.json b/worker-config.json index 6b39e3e..2c077ed 100644 --- a/worker-config.json +++ b/worker-config.json @@ -24,7 +24,8 @@ "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST" + "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", + "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" ] }, { @@ -84,7 +85,8 @@ "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST" + "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", + "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" ] }, { @@ -991,6 +993,30 @@ "description": "Enables or disables vLLM request logging", "required": false, "type": "toggle" + }, + "ENABLE_AUTO_TOOL_CHOICE": { + "env_var_name": "ENABLE_AUTO_TOOL_CHOICE", + "value": false, + "title": "Enable Auto Tool Choice", + "description": "Enables or disables auto tool choice", + "required": false, + "type": "toggle" + }, + "TOOL_CALL_PARSER": { + "env_var_name": "TOOL_CALL_PARSER", + "value": "", + "title": "Tool Call Parser", + "description": "Tool call parser", + "required": false, + "type": "select", + "options": [ + { "value": "", "label": "None" }, + { "value": "hermes", "label": "Hermes" }, + { "value": "mistral", "label": "Mistral" }, + { "value": "llama3_json", "label": "Llama3 JSON" }, + { "value": "pythonic", "label": "Pythonic" }, + { "value": "internlm", "label": "InternLM" } + ] } } } \ No newline at end of file From 8df7f41f1d2392ca19216832999716b2062a0cc0 Mon Sep 17 00:00:00 2001 From: Nikolai Kolodziej Date: Sun, 24 Nov 2024 06:55:43 +0100 Subject: [PATCH 02/10] fix: set empty `tool_call_parser` to None --- src/engine_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine_args.py b/src/engine_args.py index 485e505..4c445df 100644 --- a/src/engine_args.py +++ b/src/engine_args.py @@ -91,7 +91,7 @@ DEFAULT_ARGS = { "otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None), "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true'), "enable_auto_tool_choice": os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true', - "tool_call_parser": os.getenv('TOOL_CALL_PARSER', None) + "tool_call_parser": os.getenv('TOOL_CALL_PARSER', "") or None } def match_vllm_args(args): From 0a89394f1d4c46f6da403315fafa2242af304cdd Mon Sep 17 00:00:00 2001 From: Marut Pandya Date: Fri, 29 Nov 2024 16:29:33 -0800 Subject: [PATCH 03/10] Update worker-config.json --- worker-config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/worker-config.json b/worker-config.json index 6b39e3e..155ed45 100644 --- a/worker-config.json +++ b/worker-config.json @@ -1,7 +1,7 @@ { "versions": { "0.6.4": { - "imageName": "runpod/worker-v1-vllm:v1.6.0stable-cuda12.1.0", + "imageName": "runpod/worker-v1-vllm:v1.7.0stable-cuda12.1.0", "minimumCudaVersion": "12.1", "categories": [ { @@ -993,4 +993,4 @@ "type": "toggle" } } -} \ No newline at end of file +} From d9f54ce76a27bc7704a844f4afaf0cad84124285 Mon Sep 17 00:00:00 2001 From: Marut Pandya Date: Fri, 29 Nov 2024 16:30:45 -0800 Subject: [PATCH 04/10] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 92a6688..7d49828 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https: ### 1. UI for Deploying vLLM Worker on RunPod console: ![Demo of Deploying vLLM Worker on RunPod console with new UI](media/ui_demo.gif) -### 2. Worker vLLM `v1.6.0` with vLLM `0.6.3` now available under `stable` tags +### 2. Worker vLLM `v1.7.0` with vLLM `0.6.3` now available under `stable` tags -Update v1.6.0 is now available, use the image tag `runpod/worker-v1-vllm:v1.6.0stable-cuda12.1.0`. +Update v1.7.0 is now available, use the image tag `runpod/worker-v1-vllm:v1.7.0stable-cuda12.1.0`. ### 3. OpenAI-Compatible [Embedding Worker](https://github.com/runpod-workers/worker-infinity-embedding) Released Deploy your own OpenAI-compatible Serverless Endpoint on RunPod with multiple embedding models and fast inference for RAG and more! From 3e86d168920303ea069189fe4caaaa0361d94676 Mon Sep 17 00:00:00 2001 From: kldzj Date: Fri, 6 Dec 2024 13:08:48 +0100 Subject: [PATCH 05/10] fix: openai tool calling --- src/engine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/engine.py b/src/engine.py index 48e8aac..e53a193 100644 --- a/src/engine.py +++ b/src/engine.py @@ -17,7 +17,7 @@ from vllm.entrypoints.openai.serving_engine import BaseModelPath, LoRAModulePath from utils import DummyRequest, JobInput, BatchSize, create_error_response from constants import DEFAULT_MAX_CONCURRENCY, DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE_GROWTH_FACTOR, DEFAULT_MIN_BATCH_SIZE from tokenizer import TokenizerWrapper -from engine_args import get_engine_args +from engine_args import get_engine_args, DEFAULT_ARGS class vLLMEngine: def __init__(self, engine = None): @@ -146,6 +146,8 @@ class OpenAIvLLMEngine(vLLMEngine): base_model_paths=self.base_model_paths, response_role=self.response_role, chat_template=self.tokenizer.tokenizer.chat_template, + enable_auto_tools=DEFAULT_ARGS.enable_auto_tool_choice, + tool_parser=DEFAULT_ARGS.tool_call_parser, lora_modules=lora_modules, prompt_adapters=None, request_logger=None From b1fca5d257d291122d1123d125a3f3b7374384f3 Mon Sep 17 00:00:00 2001 From: kldzj Date: Fri, 6 Dec 2024 16:07:23 +0100 Subject: [PATCH 06/10] fix: move tool flags away from engine args --- src/engine.py | 4 ++-- src/engine_args.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/engine.py b/src/engine.py index e53a193..e839461 100644 --- a/src/engine.py +++ b/src/engine.py @@ -146,8 +146,8 @@ class OpenAIvLLMEngine(vLLMEngine): base_model_paths=self.base_model_paths, response_role=self.response_role, chat_template=self.tokenizer.tokenizer.chat_template, - enable_auto_tools=DEFAULT_ARGS.enable_auto_tool_choice, - tool_parser=DEFAULT_ARGS.tool_call_parser, + enable_auto_tools=os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true', + tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None, lora_modules=lora_modules, prompt_adapters=None, request_logger=None diff --git a/src/engine_args.py b/src/engine_args.py index 4c445df..45e50d1 100644 --- a/src/engine_args.py +++ b/src/engine_args.py @@ -89,9 +89,7 @@ DEFAULT_ARGS = { "qlora_adapter_name_or_path": os.getenv('QLORA_ADAPTER_NAME_OR_PATH', None), "disable_logprobs_during_spec_decoding": os.getenv('DISABLE_LOGPROBS_DURING_SPEC_DECODING', None), "otlp_traces_endpoint": os.getenv('OTLP_TRACES_ENDPOINT', None), - "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true'), - "enable_auto_tool_choice": os.getenv('ENABLE_AUTO_TOOL_CHOICE', 'false').lower() == 'true', - "tool_call_parser": os.getenv('TOOL_CALL_PARSER', "") or None + "use_v2_block_manager": os.getenv('USE_V2_BLOCK_MANAGER', 'true') } def match_vllm_args(args): From b7787d8ca89b367cc224c16ba378fdaddcbfa5c3 Mon Sep 17 00:00:00 2001 From: kldzj Date: Fri, 6 Dec 2024 16:29:06 +0100 Subject: [PATCH 07/10] fix: remove unused import --- src/engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine.py b/src/engine.py index e839461..f999315 100644 --- a/src/engine.py +++ b/src/engine.py @@ -17,7 +17,7 @@ from vllm.entrypoints.openai.serving_engine import BaseModelPath, LoRAModulePath from utils import DummyRequest, JobInput, BatchSize, create_error_response from constants import DEFAULT_MAX_CONCURRENCY, DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE_GROWTH_FACTOR, DEFAULT_MIN_BATCH_SIZE from tokenizer import TokenizerWrapper -from engine_args import get_engine_args, DEFAULT_ARGS +from engine_args import get_engine_args class vLLMEngine: def __init__(self, engine = None): From 06c2bb171574068b82029ba0a40f8f0c8f96c026 Mon Sep 17 00:00:00 2001 From: pandyamarut Date: Mon, 30 Dec 2024 15:20:11 -0800 Subject: [PATCH 08/10] upgrade vllm version Signed-off-by: pandyamarut --- Dockerfile | 2 +- README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 752273b..f5a05cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ python3 -m pip install --upgrade -r /requirements.txt # Install vLLM (switching back to pip installs since issues that required building fork are fixed and space optimization is not as important since caching) and FlashInfer -RUN python3 -m pip install vllm==0.6.4 && \ +RUN python3 -m pip install vllm==0.6.6.post1 && \ python3 -m pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3 # Setup for Option 2: Building the Image with the Model included diff --git a/README.md b/README.md index 7d49828..82cc1a6 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by the [vLLM](https: ### 1. UI for Deploying vLLM Worker on RunPod console: ![Demo of Deploying vLLM Worker on RunPod console with new UI](media/ui_demo.gif) -### 2. Worker vLLM `v1.7.0` with vLLM `0.6.3` now available under `stable` tags +### 2. Worker vLLM `v1.8.0` with vLLM `0.6.6` now available under `stable` tags -Update v1.7.0 is now available, use the image tag `runpod/worker-v1-vllm:v1.7.0stable-cuda12.1.0`. +Update v1.8.0 is now available, use the image tag `runpod/worker-v1-vllm:v1.8.0stable-cuda12.1.0`. ### 3. OpenAI-Compatible [Embedding Worker](https://github.com/runpod-workers/worker-infinity-embedding) Released Deploy your own OpenAI-compatible Serverless Endpoint on RunPod with multiple embedding models and fast inference for RAG and more! @@ -82,7 +82,7 @@ Below is a summary of the available RunPod Worker images, categorized by image s | CUDA Version | Stable Image Tag | Development Image Tag | Note | |--------------|-----------------------------------|-----------------------------------|----------------------------------------------------------------------| -| 12.1.0 | `runpod/worker-v1-vllm:v1.6.0stable-cuda12.1.0` | `runpod/worker-v1-vllm:v1.6.0dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. | +| 12.1.0 | `runpod/worker-v1-vllm:v1.8.0stable-cuda12.1.0` | `runpod/worker-v1-vllm:v1.8.0dev-cuda12.1.0` | When creating an Endpoint, select CUDA Version 12.3, 12.2 and 12.1 in the filter. | From 66ea8b1110cfe951ee07850185241c88d0859ac2 Mon Sep 17 00:00:00 2001 From: pandyamarut Date: Mon, 30 Dec 2024 17:13:03 -0800 Subject: [PATCH 09/10] update engine Signed-off-by: pandyamarut --- src/engine.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/engine.py b/src/engine.py index f999315..5ce48a6 100644 --- a/src/engine.py +++ b/src/engine.py @@ -150,6 +150,7 @@ class OpenAIvLLMEngine(vLLMEngine): tool_parser=os.getenv('TOOL_CALL_PARSER', "") or None, lora_modules=lora_modules, prompt_adapters=None, + chat_template_content_format="auto", request_logger=None ) self.completion_engine = OpenAIServingCompletion( From 8e3c26be14109bcfcbff1b98a89b68c8f17b97f9 Mon Sep 17 00:00:00 2001 From: pandyamarut Date: Mon, 30 Dec 2024 17:41:10 -0800 Subject: [PATCH 10/10] update worker-config Signed-off-by: pandyamarut --- worker-config.json | 121 +++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 60 deletions(-) diff --git a/worker-config.json b/worker-config.json index c150314..7845882 100644 --- a/worker-config.json +++ b/worker-config.json @@ -1,5 +1,66 @@ { "versions": { + "0.6.6": { + "imageName": "runpod/worker-v1-vllm:v1.8.0stable-cuda12.1.0", + "minimumCudaVersion": "12.1", + "categories": [ + { + "title": "LLM Settings", + "settings": [ + "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", + "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", + "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", + "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", + "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", + "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS", + "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", + "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", + "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", + "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", + "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", + "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", + "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", + "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", + "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", + "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", + "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", + "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST", + "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER" + ] + }, + { + "title": "Tokenizer Settings", + "settings": [ + "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" + ] + }, + { + "title": "System Settings", + "settings": [ + "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", + "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" + ] + }, + { + "title": "Streaming Settings", + "settings": [ + "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" + ] + }, + { + "title": "OpenAI Settings", + "settings": [ + "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" + ] + }, + { + "title": "Serverless Settings", + "settings": [ + "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" + ] + } + ] + }, "0.6.4": { "imageName": "runpod/worker-v1-vllm:v1.7.0stable-cuda12.1.0", "minimumCudaVersion": "12.1", @@ -241,66 +302,6 @@ ] } ] - }, - "0.5.5": { - "imageName": "runpod/worker-v1-vllm:v1.3.1stable-cuda12.1.0", - "minimumCudaVersion": "12.1", - "categories": [ - { - "title": "LLM Settings", - "settings": [ - "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE", - "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH", - "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND", - "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE", - "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING", - "DISABLE_SLIDING_WINDOW", "USE_V2_BLOCK_MANAGER", "NUM_LOOKAHEAD_SLOTS", - "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS", - "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA", - "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG", - "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE", - "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS", - "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL", - "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE", - "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE", - "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD", - "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA", - "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD", - "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST" - ] - }, - { - "title": "Tokenizer Settings", - "settings": [ - "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE" - ] - }, - { - "title": "System Settings", - "settings": [ - "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE", - "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE" - ] - }, - { - "title": "Streaming Settings", - "settings": [ - "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR" - ] - }, - { - "title": "OpenAI Settings", - "settings": [ - "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE" - ] - }, - { - "title": "Serverless Settings", - "settings": [ - "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS" - ] - } - ] } }, "schema": {