owen/worker-vllm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #268 from runpod-workers/fix/spec-config-0-to-none
Release / release (push) Waiting to run

Browse Source 
fix: spec config env vars should be none if zero
This commit is contained in:
chrisvela
2026-02-18 15:51:51 -06:00
committed by GitHub
parent 1606cff557 61faa8f137
commit d69cc021e8
1 changed files with 11 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/engine_args.py
+11 -6
View File
@@ -122,9 +122,14 @@ def get_speculative_config():
# Option 2: Build config from individual environment variables # Option 2: Build config from individual environment variables
spec_method = os.getenv('SPECULATIVE_METHOD') spec_method = os.getenv('SPECULATIVE_METHOD')
spec_model = os.getenv('SPECULATIVE_MODEL') spec_model = os.getenv('SPECULATIVE_MODEL')
num_spec_tokens = os.getenv('NUM_SPECULATIVE_TOKENS') _num_spec_tokens = os.getenv('NUM_SPECULATIVE_TOKENS')
ngram_max = os.getenv('NGRAM_PROMPT_LOOKUP_MAX') _ngram_max = os.getenv('NGRAM_PROMPT_LOOKUP_MAX')
ngram_min = os.getenv('NGRAM_PROMPT_LOOKUP_MIN') _ngram_min = os.getenv('NGRAM_PROMPT_LOOKUP_MIN')
# Convert numeric vars to int so '0' (hub.json default) is treated as unset
num_spec_tokens = (int(_num_spec_tokens) or None) if _num_spec_tokens else None
ngram_max = (int(_ngram_max) or None) if _ngram_max else None
ngram_min = (int(_ngram_min) or None) if _ngram_min else None
if not any([spec_method, spec_model, ngram_max]): if not any([spec_method, spec_model, ngram_max]):
return None return None
@@ -150,11 +155,11 @@ def get_speculative_config():
if spec_model: if spec_model:
config['model'] = spec_model config['model'] = spec_model
if num_spec_tokens: if num_spec_tokens:
config['num_speculative_tokens'] = int(num_spec_tokens) config['num_speculative_tokens'] = num_spec_tokens
if ngram_max: if ngram_max:
config['prompt_lookup_max'] = int(ngram_max) config['prompt_lookup_max'] = ngram_max
if ngram_min: if ngram_min:
config['prompt_lookup_min'] = int(ngram_min) config['prompt_lookup_min'] = ngram_min
draft_tp = os.getenv('SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE') draft_tp = os.getenv('SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE')
if draft_tp: if draft_tp: