diff --git a/Dockerfile b/Dockerfile index e354ac5..dcec772 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,6 @@ RUN --mount=type=cache,target=/root/.cache/pip \ ADD src . # Quick temporary updates -RUN pip install git+https://github.com/runpod/runpod-python@multijob#egg=runpod --compile +RUN pip install git+https://github.com/runpod/runpod-python@multijob2#egg=runpod --compile CMD python -u /handler.py diff --git a/src/handler.py b/src/handler.py index cd4952c..921bc43 100644 --- a/src/handler.py +++ b/src/handler.py @@ -8,6 +8,7 @@ from typing import AsyncGenerator, Dict from vllm import AsyncLLMEngine, SamplingParams, AsyncEngineArgs from vllm.utils import random_uuid import runpod +import asyncio # Prepare the model and tokenizer MODEL = 'facebook/opt-125m' @@ -27,8 +28,6 @@ engine_args = AsyncEngineArgs( # Create the vLLM asynchronous engine llm = AsyncLLMEngine.from_engine_args(engine_args) -# Run the engine for one step without inputs to ensure it's ready for handling. -llm.engine.step() def handler_fully_utilized() -> bool: # Compute pending sequences @@ -96,7 +95,7 @@ async def handler(job): prompt = job_input['prompt'] # Streaming - streaming = job_input['streaming'] or False + streaming = job_input.get('streaming', False) # Validate the inputs sampling_params = job_input.get('sampling_params', None)