diff --git a/Dockerfile b/Dockerfile index dcec772..f1b959d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,6 @@ RUN --mount=type=cache,target=/root/.cache/pip \ ADD src . # Quick temporary updates -RUN pip install git+https://github.com/runpod/runpod-python@multijob2#egg=runpod --compile +RUN pip install git+https://github.com/runpod/runpod-python@multijob2#egg=runpod --compile CMD python -u /handler.py diff --git a/src/handler.py b/src/handler.py index 921bc43..4bf7352 100644 --- a/src/handler.py +++ b/src/handler.py @@ -34,6 +34,7 @@ def handler_fully_utilized() -> bool: total_pending_sequences = len(llm.engine.scheduler.waiting) + len(llm.engine.scheduler.swapped) return total_pending_sequences > 10 + # Validation def validate_sampling_params(sampling_params): def validate_int(value, default):