From dd6bb3ee9932f5764089ca996d1df4c419e0cbfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=8E=E4=B8=96=E6=97=A0=E4=BA=89=E5=89=91=E5=AE=87?= =?UTF-8?q?=E5=AD=90?= <1530683628@qq.com> Date: Wed, 21 Oct 2026 13:33:15 +0800 Subject: [PATCH] fix(router): apply retries and fallbacks to sync text completion --- litellm/router.py | 84 ++++++++++--- .../test_router_text_completion.py | 117 ++++++++++++++++++ 2 files changed, 183 insertions(+), 18 deletions(-) create mode 100644 tests/unit/test_router/test_router_text_completion.py diff --git a/litellm/router.py b/litellm/router.py index 0ac21960be1..43184676582 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -439,6 +439,7 @@ from litellm.utils import ( ModelResponse, Rules, TextCompletionResponse, + TextCompletionStreamWrapper, function_setup, get_llm_provider, get_non_default_completion_params, @@ -5825,32 +5826,79 @@ class Router: is_async: bool | None = False, **kwargs, ): - messages: Final = [{"role": "user", "content": prompt}] - try: - kwargs["model"] = model - kwargs["prompt"] = prompt - kwargs["num_retries"] = kwargs.get("num_retries", self.num_retries) - kwargs.setdefault("metadata", {}).update({"model_group": model}) + kwargs["model"] = model + kwargs["prompt"] = prompt + kwargs["original_function"] = self._text_completion + self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs) # pyright: ignore[reportUnknownMemberType] # dynamic router kwargs - # pick the one that is available (lowest TPM/RPM) - deployment: Final = self.get_available_deployment( + return self.function_with_fallbacks(**kwargs) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # dynamic fallback + + def _text_completion( + self, + model: str, + prompt: str, + **kwargs: object, # kwargs-ok: router fallback callbacks receive provider-specific endpoint options + ) -> TextCompletionResponse | ModelResponse | CustomStreamWrapper | TextCompletionStreamWrapper: + model_name = None # rebind-ok: pre-init so the except block can count failures before a deployment is selected + deployment = None # rebind-ok: pre-init so the except block can stamp errors when selection fails + try: + verbose_router_logger.debug("Inside _text_completion()- model: %s; kwargs: %s", model, kwargs) + deployment = self.get_available_deployment( # rebind-ok: set after a deployment is selected model=model, - messages=messages, - specific_deployment=kwargs.pop("specific_deployment", None), + messages=[{"role": "user", "content": prompt}], + specific_deployment=cast( # cast-ok: router-only option has no typed schema + bool | None, kwargs.pop("specific_deployment", None) + ), request_kwargs=kwargs, ) + self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, splits_stream_timeout=True) data: Final = deployment["litellm_params"].copy() - for k, v in self.default_litellm_params.items(): - if k not in kwargs: # prioritize model-specific params > default router params - kwargs[k] = v - elif k == "metadata": - kwargs[k].update(v) + model_name = data["model"] # pyright: ignore[reportUnknownVariableType] # rebind-ok: dynamic deployment payload - # call via litellm.completion() - return litellm.text_completion(**{**data, "prompt": prompt, "caching": self.cache_responses, **kwargs}) + potential_model_client: Final = self._get_client(deployment=deployment, kwargs=kwargs) + dynamic_api_key: Final = kwargs.get("api_key", None) + model_client: Final = ( + None + if ( + dynamic_api_key is not None + and potential_model_client is not None + and dynamic_api_key != potential_model_client.api_key + ) + else potential_model_client + ) + + self.total_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # untyped call counter + + ### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit) + ## only run if model group given, not model id + if model in self.model_names or not self.has_model_id(model): + self.routing_strategy_pre_call_checks(deployment=deployment) # pyright: ignore[reportUnknownMemberType] # untyped deployment + + response: Final = litellm.text_completion( + **without_router_only_kwargs( # pyright: ignore[reportUnknownArgumentType] # dynamic request payload + { + **data, + "prompt": prompt, + "caching": self.cache_responses, + "client": model_client, + **kwargs, + } + ) + ) except Exception as e: - raise e + verbose_router_logger.info("litellm.text_completion(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) # pyright: ignore[reportUnknownArgumentType] # dynamic deployment + if model_name is not None: + self.fail_calls[model_name] += 1 + if deployment is not None: + self._set_deployment_num_retries_on_exception(e, deployment) + self._stamp_failed_deployment_id_with_effective_model_info(e, deployment, kwargs) # pyright: ignore[reportUnknownArgumentType] # dynamic deployment + self._stamp_retry_skip_deployment_id(e, kwargs) + raise + else: + self.success_calls[model_name] += 1 + verbose_router_logger.info("litellm.text_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) # pyright: ignore[reportUnknownArgumentType] # dynamic deployment + return response async def atext_completion( self, diff --git a/tests/unit/test_router/test_router_text_completion.py b/tests/unit/test_router/test_router_text_completion.py new file mode 100644 index 00000000000..33549cb80a7 --- /dev/null +++ b/tests/unit/test_router/test_router_text_completion.py @@ -0,0 +1,117 @@ +from typing import Any +from unittest.mock import MagicMock, patch + +import litellm +from litellm import Router + + +def test_text_completion_uses_deployment_model_name(): + router = Router( + model_list=[ + { + "model_name": "my-instruct", + "litellm_params": { + "model": "text-completion-openai/gpt-3.5-turbo-instruct", + "api_key": "fake-key", + }, + } + ], + num_retries=0, + ) + expected_response = MagicMock() + + with patch("litellm.text_completion", return_value=expected_response) as mock_text_completion: + response = router._text_completion(model="my-instruct", prompt="Hello") + + assert response is expected_response + assert mock_text_completion.call_args.kwargs["model"] == "text-completion-openai/gpt-3.5-turbo-instruct" + + +def test_text_completion_falls_back_to_secondary_model(): + router = Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "text-completion-openai/primary-model", + "api_key": "fake-key", + }, + }, + { + "model_name": "backup", + "litellm_params": { + "model": "text-completion-openai/backup-model", + "api_key": "fake-key", + }, + }, + ], + fallbacks=[{"primary": ["backup"]}], + num_retries=0, + ) + expected_response = MagicMock() + called_models = [] + + def fake_text_completion(**kwargs: Any) -> Any: + called_models.append(kwargs["model"]) + if kwargs["model"] == "text-completion-openai/primary-model": + raise litellm.InternalServerError( + message="primary deployment is unavailable", + llm_provider="text-completion-openai", + model="primary-model", + ) + return expected_response + + with patch("litellm.text_completion", side_effect=fake_text_completion): + response = router.text_completion(model="primary", prompt="Hello") + + assert response is expected_response + assert called_models == [ + "text-completion-openai/primary-model", + "text-completion-openai/backup-model", + ] + + +def test_text_completion_retries_after_deployment_failure(): + router = Router( + model_list=[ + { + "model_name": "instruct", + "litellm_params": { + "model": "text-completion-openai/primary-model", + "api_key": "fake-key", + }, + }, + { + "model_name": "instruct", + "litellm_params": { + "model": "text-completion-openai/backup-model", + "api_key": "fake-key", + }, + }, + ], + num_retries=1, + ) + expected_response = MagicMock() + called_models = [] + + def fake_text_completion(**kwargs: Any) -> Any: + called_models.append(kwargs["model"]) + if len(called_models) == 1: + raise litellm.InternalServerError( + message="first deployment is unavailable", + llm_provider="text-completion-openai", + model="primary-model", + ) + return expected_response + + with patch("litellm.text_completion", side_effect=fake_text_completion): + response = router.text_completion(model="instruct", prompt="Hello") + + assert response is expected_response + assert len(called_models) == 2 + assert set(called_models).issubset( + { + "text-completion-openai/primary-model", + "text-completion-openai/backup-model", + } + )