Borrow vLLM concurrenccy logic a little.

2025-09-11 00:31:09 -05:00
parent 0df1db40e3
commit 51d38adcf3
2 changed files with 31 additions and 7 deletions
@@ -13,13 +13,33 @@
    "env": [
      {
        "key": "MODEL_NAME",
-          "input": {
-            "name": "Model Name",
-            "type": "string",
-            "description": "Name of a model to preload",
-            "default": "phi3",
-            "advanced": false
-          }
+        "input": {
+          "name": "Model Name",
+          "type": "string",
+          "description": "Name of a model to preload",
+          "default": "phi3",
+          "advanced": false
+        }
+      },
+      {
+        "key": "MAX_CONCURRENCY",
+        "input": {
+          "name": "Max Concurrency",
+          "type": "number",
+          "description": "Maximum number of concurrent requests to handle (default: 8)",
+          "default": 8,
+          "advanced": true
+        }
+      },
+      {
+        "key": "OLLAMA_NUM_PARALLEL",
+        "input": {
+          "name": "Parallel Requests",
+          "type": "string",
+          "description": "Maximum number of concurrent requests to handle (default: 4 or 1)",
+          "default": "",
+          "advanced": true
+        }
      }
    ]
  }