diff --git a/builder/requirements.txt b/builder/requirements.txt index 7ec7932..a52d309 100644 --- a/builder/requirements.txt +++ b/builder/requirements.txt @@ -11,5 +11,5 @@ hf-transfer transformers>=4.56.0,<5 bitsandbytes>=0.45.0 kernels -torch>=2.10.0 +torch>=2.9.1 torch-c-dlpack-ext \ No newline at end of file diff --git a/src/engine.py b/src/engine.py index 6d3c859..eb74b21 100644 --- a/src/engine.py +++ b/src/engine.py @@ -176,7 +176,7 @@ class vLLMEngine: class OpenAIvLLMEngine(vLLMEngine): def __init__(self, vllm_engine): super().__init__(vllm_engine) - self.served_model_name = os.getenv("OPENAI_SERVED_MODEL_NAME_OVERRIDE") or self.engine_args.model + self.served_model_name = os.getenv("OPENAI_SERVED_MODEL_NAME_OVERRIDE") or self.engine_args.served_model_name or self.engine_args.model self.response_role = os.getenv("OPENAI_RESPONSE_ROLE") or "assistant" self.lora_adapters = self._load_lora_adapters() self._engines_initialized = False @@ -215,7 +215,7 @@ class OpenAIvLLMEngine(vLLMEngine): async def _initialize_engines(self): logging.info("Initializing OpenAI serving engines...") self.base_model_paths = [ - BaseModelPath(name=self.engine_args.model, model_path=self.engine_args.model) + BaseModelPath(name=self.served_model_name, model_path=self.engine_args.model) ] self.serving_models = OpenAIServingModels(