From fed20b9db92543f90c87462186e7f94554a54491 Mon Sep 17 00:00:00 2001 From: Lalith Vardhan Date: Tue, 20 Oct 2026 19:52:26 -0700 Subject: [PATCH 1/3] fix(router): route sync text_completion through deployment selection and fallbacks --- litellm/router.py | 39 +++++++++++----- .../unit/test_router/test_router_endpoints.py | 46 +++++++++++++++++++ 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 1ce1e0a5a70..596b8f95896 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5724,31 +5724,46 @@ class Router: is_async: bool | None = False, **kwargs, ): - messages: Final = [{"role": "user", "content": prompt}] try: kwargs["model"] = model kwargs["prompt"] = prompt - kwargs["num_retries"] = kwargs.get("num_retries", self.num_retries) - kwargs.setdefault("metadata", {}).update({"model_group": model}) + kwargs["original_function"] = self._text_completion + self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs) + return self.function_with_fallbacks(**kwargs) + except Exception as e: + raise e - # pick the one that is available (lowest TPM/RPM) + def _text_completion(self, model: str, prompt: str, **kwargs): + model_name = None + try: + verbose_router_logger.debug("Inside _text_completion()- model: %s; kwargs: %s", model, kwargs) deployment: Final = self.get_available_deployment( model=model, - messages=messages, + messages=[{"role": "user", "content": prompt}], specific_deployment=kwargs.pop("specific_deployment", None), request_kwargs=kwargs, ) + self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) data: Final = deployment["litellm_params"].copy() - for k, v in self.default_litellm_params.items(): - if k not in kwargs: # prioritize model-specific params > default router params - kwargs[k] = v - elif k == "metadata": - kwargs[k].update(v) + model_name = data["model"] + self.total_calls[model_name] += 1 - # call via litellm.completion() - return litellm.text_completion(**{**data, "prompt": prompt, "caching": self.cache_responses, **kwargs}) + ### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit) + self.routing_strategy_pre_call_checks(deployment=deployment) + + # kwargs no longer carries the model group name, so the deployment's model is used + response: Final = litellm.text_completion( + **{**data, "prompt": prompt, "caching": self.cache_responses, **kwargs} + ) + self.success_calls[model_name] += 1 + verbose_router_logger.info("litellm.text_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) + return response except Exception as e: + verbose_router_logger.info("litellm.text_completion(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) + if model_name is not None: + self.fail_calls[model_name] += 1 + self._stamp_retry_skip_deployment_id(e, kwargs) raise e async def atext_completion( diff --git a/tests/unit/test_router/test_router_endpoints.py b/tests/unit/test_router/test_router_endpoints.py index 815b32b33f7..4cb55bb6383 100644 --- a/tests/unit/test_router/test_router_endpoints.py +++ b/tests/unit/test_router/test_router_endpoints.py @@ -1259,3 +1259,49 @@ def test_router_model_group_encrypted_content_affinity_callback_registration(): assert encrypted_content_callbacks[0].router is router finally: router.discard() + + +def test_sync_text_completion_calls_the_deployment_model_not_the_group_alias(): + router = Router( + model_list=[ + { + "model_name": "my-instruct-alias", + "litellm_params": {"model": "gpt-3.5-turbo-instruct", "api_key": "fake-key"}, + } + ] + ) + + with patch.object(litellm, "text_completion", wraps=litellm.text_completion) as text_completion_spy: + response = router.text_completion(model="my-instruct-alias", prompt="hi", mock_response="ok") + + assert response.choices[0].text == "ok", response + assert text_completion_spy.call_args.kwargs["model"] == "gpt-3.5-turbo-instruct", text_completion_spy.call_args + + +def test_sync_text_completion_falls_back_like_async(): + router = Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "gpt-3.5-turbo-instruct", + "api_key": "fake-key", + "mock_response": "litellm.RateLimitError", + }, + }, + { + "model_name": "backup", + "litellm_params": { + "model": "gpt-3.5-turbo-instruct", + "api_key": "fake-key", + "mock_response": "from backup", + }, + }, + ], + fallbacks=[{"primary": ["backup"]}], + num_retries=0, + ) + + response = router.text_completion(model="primary", prompt="hi") + + assert response.choices[0].text == "from backup", response From 11835a2a9ff6a0190997c47d372ee07bf5be3b07 Mon Sep 17 00:00:00 2001 From: Lalith Vardhan Date: Fri, 23 Oct 2026 10:16:11 -0700 Subject: [PATCH 2/3] fix(router): satisfy lint gates for sync text_completion fix (ruff-strict, router coverage, types) --- litellm/router.py | 33 ++++++++++--------- .../unit/test_router/test_router_endpoints.py | 30 +++++++++++++++++ 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 596b8f95896..adc4ab6f4c3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -438,6 +438,7 @@ from litellm.utils import ( ModelResponse, Rules, TextCompletionResponse, + TextCompletionStreamWrapper, function_setup, get_llm_provider, get_non_default_completion_params, @@ -5724,23 +5725,25 @@ class Router: is_async: bool | None = False, **kwargs, ): - try: - kwargs["model"] = model - kwargs["prompt"] = prompt - kwargs["original_function"] = self._text_completion - self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs) - return self.function_with_fallbacks(**kwargs) - except Exception as e: - raise e + kwargs["model"] = model + kwargs["prompt"] = prompt + kwargs["original_function"] = self._text_completion + self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs) + return self.function_with_fallbacks(**kwargs) - def _text_completion(self, model: str, prompt: str, **kwargs): - model_name = None + def _text_completion( + self, + model: str, + prompt: str, + **kwargs: object, # kwargs-ok: forwarded verbatim to litellm.text_completion, like _completion + ) -> TextCompletionResponse | TextCompletionStreamWrapper | ModelResponse | CustomStreamWrapper: + model_name: str | None = None try: verbose_router_logger.debug("Inside _text_completion()- model: %s; kwargs: %s", model, kwargs) deployment: Final = self.get_available_deployment( model=model, messages=[{"role": "user", "content": prompt}], - specific_deployment=kwargs.pop("specific_deployment", None), + specific_deployment=kwargs.pop("specific_deployment", None) is True, request_kwargs=kwargs, ) self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) @@ -5756,15 +5759,15 @@ class Router: response: Final = litellm.text_completion( **{**data, "prompt": prompt, "caching": self.cache_responses, **kwargs} ) - self.success_calls[model_name] += 1 - verbose_router_logger.info("litellm.text_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) - return response except Exception as e: verbose_router_logger.info("litellm.text_completion(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) if model_name is not None: self.fail_calls[model_name] += 1 self._stamp_retry_skip_deployment_id(e, kwargs) - raise e + raise + self.success_calls[model_name] += 1 + verbose_router_logger.info("litellm.text_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) + return response async def atext_completion( self, diff --git a/tests/unit/test_router/test_router_endpoints.py b/tests/unit/test_router/test_router_endpoints.py index 4cb55bb6383..7fe111f5356 100644 --- a/tests/unit/test_router/test_router_endpoints.py +++ b/tests/unit/test_router/test_router_endpoints.py @@ -1305,3 +1305,33 @@ def test_sync_text_completion_falls_back_like_async(): response = router.text_completion(model="primary", prompt="hi") assert response.choices[0].text == "from backup", response + + +def test_sync_text_completion_deployment_call_updates_call_counters(): + router = Router( + model_list=[ + { + "model_name": "ok-group", + "litellm_params": {"model": "gpt-3.5-turbo-instruct", "api_key": "fake-key", "mock_response": "ok"}, + }, + { + "model_name": "failing-group", + "litellm_params": { + "model": "openai/davinci-002", + "api_key": "fake-key", + "mock_response": "litellm.RateLimitError", + }, + }, + ], + num_retries=0, + ) + + response = router._text_completion(model="ok-group", prompt="hi") + with pytest.raises(litellm.RateLimitError): + router._text_completion(model="failing-group", prompt="hi") + + assert response.choices[0].text == "ok", response + assert router.total_calls["gpt-3.5-turbo-instruct"] == 1, router.total_calls + assert router.success_calls["gpt-3.5-turbo-instruct"] == 1, router.success_calls + assert router.total_calls["openai/davinci-002"] == 1, router.total_calls + assert router.fail_calls["openai/davinci-002"] == 1, router.fail_calls From 50828b36a41175f5911ad28005eba1b3cf793172 Mon Sep 17 00:00:00 2001 From: Lalith Vardhan Date: Sun, 25 Oct 2026 09:51:09 -0700 Subject: [PATCH 3/3] fix(router): give model_name rebinds in _text_completion a rebind-ok reason (LIT010 gate) --- litellm/router.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index adc4ab6f4c3..98e1c1d51f1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5737,7 +5737,7 @@ class Router: prompt: str, **kwargs: object, # kwargs-ok: forwarded verbatim to litellm.text_completion, like _completion ) -> TextCompletionResponse | TextCompletionStreamWrapper | ModelResponse | CustomStreamWrapper: - model_name: str | None = None + model_name: str | None = None # rebind-ok: set once the deployment is picked; read in except for fail_calls try: verbose_router_logger.debug("Inside _text_completion()- model: %s; kwargs: %s", model, kwargs) deployment: Final = self.get_available_deployment( @@ -5749,7 +5749,7 @@ class Router: self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) data: Final = deployment["litellm_params"].copy() - model_name = data["model"] + model_name = data["model"] # rebind-ok: assigned inside try so except can attribute the failure self.total_calls[model_name] += 1 ### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit)