diff --git a/src/engine_args.py b/src/engine_args.py index 66ed122..1d99005 100644 --- a/src/engine_args.py +++ b/src/engine_args.py @@ -147,8 +147,8 @@ def get_engine_args(): # Rename and match to vllm args args = match_vllm_args(args) - if args.load_format=="bitsandbytes": - args.quantization = args.load_format + if args.get("load_format") == "bitsandbytes": + args["quantization"] = args["load_format"] # Set tensor parallel size and max parallel loading workers if more than 1 GPU is available num_gpus = device_count()