scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

refactor(router): read a deployment's declared prefill support without an empty except

Browse source
This commit is contained in:
mateo-berri 2026-10-01 17:31:16 -07:00
parent 6e52b45dbf
commit 430d931ff8
2 changed files with 14 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
litellm/router_utils/pre_call_checks/continuation_prefill_check.py
View file

@ -24,18 +24,23 @@ MID_STREAM_CONTINUATION_MARKER: Final = _ContinuationMarker()
_STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
def _declared_prefill_support(deployment_map: Mapping[str, object]) -> bool | None:
try:
model_info: Final = _STR_KEYED_DICT_ADAPTER.validate_python(deployment_map.get("model_info"))
except ValidationError:
return None
declared: Final = model_info.get("supports_assistant_prefill")
return declared if isinstance(declared, bool) else None
def _deployment_supports_prefill(deployment: object) -> bool:
try:
deployment_map: Final = _STR_KEYED_DICT_ADAPTER.validate_python(deployment)
except ValidationError:
return False
try:
model_info: Final = _STR_KEYED_DICT_ADAPTER.validate_python(deployment_map.get("model_info"))
declared: Final = model_info.get("supports_assistant_prefill")
if isinstance(declared, bool):
return declared
except ValidationError:
pass
declared: Final = _declared_prefill_support(deployment_map)
if declared is not None:
return declared
try:
litellm_params: Final = _STR_KEYED_DICT_ADAPTER.validate_python(deployment_map.get("litellm_params"))
except ValidationError:

2
tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py
View file

@ -32,6 +32,8 @@ def _deployment(model: str, dep_id: str) -> dict:
def test_deployment_supports_prefill_reads_capability():
assert _deployment_supports_prefill(_deployment(PREFILL_MODEL, "a")) is True
assert _deployment_supports_prefill(_deployment(NON_PREFILL_MODEL, "b")) is False
assert _deployment_supports_prefill({"litellm_params": {"model": PREFILL_MODEL}}) is True
assert _deployment_supports_prefill({"litellm_params": {"model": PREFILL_MODEL}, "model_info": "bogus"}) is True
def test_deployment_model_info_override_wins_over_cost_map():