fix: kv_transfer_config is dataclass not json, fix for env var

This commit is contained in:
velaraptor-runpod
2026-04-09 14:33:11 -05:00
parent 9035b0e07f
commit dc4ad7ddeb
+32 -4
View File
@@ -1,3 +1,4 @@
import ast
import os
import json
import logging
@@ -113,6 +114,17 @@ def _convert_env_value_to_field_type(value: str, field_name: str, field_type: ty
except (json.JSONDecodeError, TypeError):
pass
return tuple(elem_type(x.strip()) for x in str(val).split(",") if x.strip())
# For dataclass/complex types, try JSON then Python literal parsing to dict
try:
return json.loads(val)
except (json.JSONDecodeError, TypeError):
pass
try:
parsed = ast.literal_eval(val)
if isinstance(parsed, (dict, list)):
return parsed
except (ValueError, SyntaxError):
pass
# Fallback: try int, float, then str
try:
return int(val)
@@ -404,19 +416,35 @@ def get_engine_args():
# LMCache requires HMA to be disabled
try:
_kv_transfer = args.get("kv_transfer_config")
if isinstance(_kv_transfer, str):
parsed = None
try:
parsed = json.loads(_kv_transfer)
except (json.JSONDecodeError, TypeError):
pass
if parsed is None:
try:
result = ast.literal_eval(_kv_transfer)
if isinstance(result, dict):
parsed = result
except (ValueError, SyntaxError):
pass
if parsed is not None:
_kv_transfer = parsed
args["kv_transfer_config"] = _kv_transfer
_kv_offload = args.get("kv_offloading_backend")
lmcache_detected = _kv_offload == "lmcache" or (
lmcache_via_offload = _kv_offload == "lmcache"
lmcache_via_transfer = (
isinstance(_kv_transfer, dict)
and isinstance(_kv_transfer.get("kv_connector"), str)
and "lmcache" in _kv_transfer.get("kv_connector", "").lower()
)
lmcache_detected = lmcache_via_offload or lmcache_via_transfer
if lmcache_detected and not args.get("disable_hybrid_kv_cache_manager"):
args["disable_hybrid_kv_cache_manager"] = True
args["kv_offloading_backend"] = None
args["kv_transfer_config"] = None
logging.info("LMCache detected: automatically setting disable_hybrid_kv_cache_manager=True and clearing conflicting settings")
logging.info("LMCache detected: automatically setting disable_hybrid_kv_cache_manager=True")
elif lmcache_detected and args.get("disable_hybrid_kv_cache_manager") is False:
logging.warning(
"LMCache configuration detected but disabled: "