From 22d0b1e44315cda2a4b2bb549180a6d7c16acaf2 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Sat, 31 Oct 2026 15:58:34 -0700 Subject: [PATCH] fix(router): restore four base behaviors the RoutingContext refactor changed (#49395) --- .../auto_router/litellm_encoder.py | 4 +-- .../complexity_router/jev_classifier.py | 1 + litellm/router_strategy/tag_based_routing.py | 4 ++- litellm/router_utils/handle_error.py | 5 +++- .../complexity_router/test_jev_classifier.py | 21 +++++++++++++++ .../router_strategy/test_litellm_encoder.py | 16 ++++++++++++ .../test_router_tag_routing.py | 26 +++++++++++++++++++ tests/unit/router_utils/test_handle_error.py | 7 +++++ tests/unit/test_router/core/test_endpoints.py | 2 ++ 9 files changed, 82 insertions(+), 4 deletions(-) diff --git a/litellm/router_strategy/auto_router/litellm_encoder.py b/litellm/router_strategy/auto_router/litellm_encoder.py index 340efb509eb..743a0b97fd2 100644 --- a/litellm/router_strategy/auto_router/litellm_encoder.py +++ b/litellm/router_strategy/auto_router/litellm_encoder.py @@ -14,13 +14,13 @@ else: RoutingContext = Any -def litellm_to_list(embeds: litellm.EmbeddingResponse) -> list[list[float]]: +def litellm_to_list(embeds: object) -> list[list[float]]: """Convert a LiteLLM embedding response to a list of embeddings. :param embeds: The LiteLLM embedding response. :return: A list of embeddings. """ - if not embeds or not embeds.data: + if not embeds or not isinstance(embeds, litellm.EmbeddingResponse) or not embeds.data: raise ValueError("No embeddings found in LiteLLM embedding response.") return [x["embedding"] for x in embeds.data] diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py index 89e9dec96f1..bb7d2e91cd3 100644 --- a/litellm/router_strategy/complexity_router/jev_classifier.py +++ b/litellm/router_strategy/complexity_router/jev_classifier.py @@ -68,6 +68,7 @@ class JevBudgetedRequest: telemetry: StandardLoggingRoutingDecision +@renamed_router_param("router", "context") def jev_input_targets(context: "RoutingContext", config: "OpenSourceClassifierConfig") -> tuple[JevInputTarget, ...]: from litellm.types.router import Deployment diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index dca717100bf..f2aafa5b7df 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -392,9 +392,11 @@ def _resolve_constraint_only_pool( def _all_deployments_or_fallback( ctx: "RoutingContext", - model: str, + model: str | None, fallback: _DeploymentPool, ) -> Sequence[_DeploymentLike | DeploymentTypedDict] | Mapping[_DeploymentLike, object]: + if model is None: # a group of no name has no deployments, it is not a failed lookup + return () try: return ctx.registry().deployments(model, source="all") except Exception: # noqa: BLE001 # fail safe toward today's healthy-only behavior on lookup errors diff --git a/litellm/router_utils/handle_error.py b/litellm/router_utils/handle_error.py index ad502fe68aa..6033024123f 100644 --- a/litellm/router_utils/handle_error.py +++ b/litellm/router_utils/handle_error.py @@ -23,7 +23,7 @@ else: @renamed_router_param("litellm_router_instance") async def send_llm_exception_alert( - ctx: "RoutingContext", + ctx: "RoutingContext | None", request_kwargs: dict, error_traceback_str: str, original_exception, @@ -39,6 +39,9 @@ async def send_llm_exception_alert( Returns: None """ + if ctx is None: + return + slack_alerting_logger: Final = ctx.observers().slack_alerting_logger if slack_alerting_logger is None: return diff --git a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py index abbb1ed80d0..bcbf69e14a6 100644 --- a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py +++ b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py @@ -1316,3 +1316,24 @@ async def test_jev_open_breaker_preserves_input_budget_in_fallback_decision(fall transport.handle_async_request.assert_awaited_once() finally: await handler.client.aclose() + + +def test_jev_input_targets_still_takes_a_router_with_a_deprecation_warning() -> None: + from litellm.router_strategy.complexity_router.jev_classifier import jev_input_targets + + router = litellm.Router( + model_list=[ + { + "model_name": "decision-deployment", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, + "model_info": {"id": "d1", "max_input_tokens": 1234}, + } + ] + ) + config = JevClassifierConfig.model_validate({"deployment_name": "decision-deployment"}) + + with pytest.warns(DeprecationWarning, match="pass context"): + old_way = jev_input_targets(router, config) + + assert old_way == jev_input_targets(router._core_ctx, config) + assert [target.max_tokens for target in old_way] == [1234] diff --git a/tests/unit/router_strategy/test_litellm_encoder.py b/tests/unit/router_strategy/test_litellm_encoder.py index d950025afdb..f12aa7cdb6a 100644 --- a/tests/unit/router_strategy/test_litellm_encoder.py +++ b/tests/unit/router_strategy/test_litellm_encoder.py @@ -134,3 +134,19 @@ class TestEmbeddingInputCap: _encoder(router, max_input_chars=20).encode_queries(["route me please, then a huge pasted file"]) assert router.embedded_inputs == [["route me please, the"]] + + +@pytest.mark.parametrize( + "embeds", + [ + {"data": [{"embedding": [0.1]}]}, + [{"embedding": [0.1]}], + type("HasData", (), {"data": [{"embedding": [0.1]}]})(), + ], + ids=["dict", "list", "object-with-data"], +) +def test_litellm_to_list_rejects_anything_but_an_embedding_response(embeds: object) -> None: + from litellm.router_strategy.auto_router.litellm_encoder import litellm_to_list + + with pytest.raises(ValueError, match="No embeddings found"): + litellm_to_list(embeds) diff --git a/tests/unit/router_strategy/test_router_tag_routing.py b/tests/unit/router_strategy/test_router_tag_routing.py index b4e29df8116..0c19cfc72f6 100644 --- a/tests/unit/router_strategy/test_router_tag_routing.py +++ b/tests/unit/router_strategy/test_router_tag_routing.py @@ -1225,6 +1225,32 @@ def test_chain_tag_filtering_override_falls_back_to_healthy_deployments_on_looku assert _chain_tag_filtering_override(_BrokenCtx(), "gpt-4", healthy_deployments) is False +@pytest.mark.asyncio() +async def test_get_deployments_for_tag_without_a_model_keeps_the_router_wide_setting(): + # model=None names no group, so no group-level enable_tag_filtering applies: the router-wide + # setting (off here) decides, even when a healthy deployment opts in on its own + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o", "api_key": "sk-test", "tags": ["teamA"]}, + "model_info": {"id": "a", "enable_tag_filtering": True}, + } + ] + ) + healthy_deployments = router.registry().deployments("gpt-4", source="all") + + picked = await get_deployments_for_tag( + router._core_ctx, + None, # pyright: ignore[reportArgumentType] # a direct caller passing no model + healthy_deployments, + request_kwargs={"metadata": {"tags": ["zzz"]}}, + request_deployments=healthy_deployments, + ) + + assert [d["model_info"]["id"] for d in picked] == ["a"] + + @pytest.mark.asyncio() async def test_chain_enable_tag_filtering_true_overrides_router_level_false(): # Router-wide tag filtering is off; this model group opts in on its own via diff --git a/tests/unit/router_utils/test_handle_error.py b/tests/unit/router_utils/test_handle_error.py index 151fb0bfc68..22bd6e74190 100644 --- a/tests/unit/router_utils/test_handle_error.py +++ b/tests/unit/router_utils/test_handle_error.py @@ -273,3 +273,10 @@ def setup_and_teardown(): yield loop.close() asyncio.set_event_loop(None) + + +@pytest.mark.asyncio +async def test_send_llm_exception_alert_without_a_router_does_nothing(): + from litellm.router_utils.handle_error import send_llm_exception_alert + + assert await send_llm_exception_alert(None, {"model": "gpt-5-mini"}, "traceback", Exception("boom")) is None diff --git a/tests/unit/test_router/core/test_endpoints.py b/tests/unit/test_router/core/test_endpoints.py index b2ad602c4c6..c0c4c8cf6dd 100644 --- a/tests/unit/test_router/core/test_endpoints.py +++ b/tests/unit/test_router/core/test_endpoints.py @@ -414,6 +414,8 @@ def _scripted_stream(*texts: str) -> Callable[..., CustomStreamWrapper]: async def chunks() -> AsyncIterator[ModelResponseStream]: for text in texts: yield ModelResponseStream(choices=[{"index": 0, "delta": {"role": "assistant", "content": text}}]) + # A stream that ends without a finish reason fails since #47665. + yield ModelResponseStream(choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}]) yield ModelResponseStream(choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}]) logging_obj: Final = LiteLLMLogging(