From 4bcaafd453b2b72817a05eed10cc9a1b831dc480 Mon Sep 17 00:00:00 2001 From: jagjeet-singh-23 Date: Sun, 5 Jul 2026 14:24:26 +0530 Subject: [PATCH 1/6] fix(budget-limiter): derive custom_llm_provider from model string for /v1/messages and /v1/embeddings routes When provider_budget_config is enabled, async_log_success_event in RouterBudgetLimiting raised ValueError for /v1/messages and /v1/embeddings requests because those route handlers do not inject custom_llm_provider into litellm_params (unlike /v1/chat/completions which sets it correctly). Fix: before raising, attempt to derive the provider from the model string using the existing litellm.get_llm_provider() helper already present in the same file. Falls back to ValueError only if derivation also fails. Fixes #26701 --- litellm/router_strategy/budget_limiter.py | 19 +++ ...test_budget_limiter_provider_derivation.py | 113 ++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 tests/test_litellm/router_strategy/test_budget_limiter_provider_derivation.py diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index 067f38ab11c..31240cf1401 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -408,6 +408,25 @@ class RouterBudgetLimiting(CustomLogger): response_cost: float = standard_logging_payload.get("response_cost", 0) model_id: str = str(standard_logging_payload.get("model_id", "")) custom_llm_provider: str = kwargs.get("litellm_params", {}).get("custom_llm_provider", None) + if custom_llm_provider is None: + # /v1/messages and /v1/embeddings routes do not inject + # custom_llm_provider into litellm_params the way + # /v1/chat/completions does. Derive it from the model string + # so budget tracking is not silently skipped for those routes. + # See: https://github.com/BerriAI/litellm/issues/26701 + _litellm_params = kwargs.get("litellm_params") or {} + _model = ( + _litellm_params.get("model", "") + if isinstance(_litellm_params, dict) + else getattr(_litellm_params, "model", "") or "" + ) + try: + _, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=str(_model), + litellm_params=_LiteLLMParamsDictView(_litellm_params if isinstance(_litellm_params, dict) else {}), + ) + except Exception: + custom_llm_provider = None if custom_llm_provider is None: raise ValueError("custom_llm_provider is required") diff --git a/tests/test_litellm/router_strategy/test_budget_limiter_provider_derivation.py b/tests/test_litellm/router_strategy/test_budget_limiter_provider_derivation.py new file mode 100644 index 00000000000..a01a004c22e --- /dev/null +++ b/tests/test_litellm/router_strategy/test_budget_limiter_provider_derivation.py @@ -0,0 +1,113 @@ +""" +Regression tests for #26701 + +/v1/messages and /v1/embeddings routes don't inject custom_llm_provider into +litellm_params (unlike /v1/chat/completions). async_log_success_event must +derive the provider from the model string instead of raising ValueError. +""" + +import pytest +from unittest.mock import AsyncMock, patch + +from litellm.caching.caching import DualCache +from litellm.router_strategy.budget_limiter import RouterBudgetLimiting + + +def _make_limiter(): + return RouterBudgetLimiting( + dual_cache=DualCache(), + provider_budget_config={ + "anthropic": {"budget_limit": 5.0, "time_period": "24h"} + }, + ) + + +def _kwargs_without_custom_provider(model_str: str) -> dict: + """Simulates what /v1/messages and /v1/embeddings produce — no custom_llm_provider.""" + return { + "model": model_str, + "litellm_params": { + "model": model_str, + # custom_llm_provider deliberately absent — this is the bug scenario + }, + "standard_logging_object": { + "response_cost": 0.001, + "model_id": "deployment-abc123", + }, + } + + +@pytest.mark.asyncio +async def test_messages_route_no_custom_provider_does_not_raise(): + """ + /v1/messages kwargs missing custom_llm_provider must not raise ValueError. + Provider should be derived from the model string 'anthropic/...' instead. + """ + limiter = _make_limiter() + kwargs = _kwargs_without_custom_provider("anthropic/claude-haiku-4-5-20251001") + + with patch.object(limiter, "_increment_spend_for_key", new_callable=AsyncMock) as mock_increment, \ + patch.object(limiter, "_get_budget_config_for_deployment", return_value=None), \ + patch.object(limiter, "_get_budget_config_for_tag", return_value=None): + + # Before the fix this raises: ValueError("custom_llm_provider is required") + await limiter.async_log_success_event( + kwargs=kwargs, + response_obj=None, + start_time=None, + end_time=None, + ) + + # Provider was correctly derived → spend increment was called + mock_increment.assert_called_once() + + +@pytest.mark.asyncio +async def test_embeddings_route_no_custom_provider_does_not_raise(): + """/v1/embeddings has the same gap — openai provider should be derived cleanly.""" + limiter = _make_limiter() + kwargs = _kwargs_without_custom_provider("openai/text-embedding-3-small") + + with patch.object(limiter, "_increment_spend_for_key", new_callable=AsyncMock), \ + patch.object(limiter, "_get_budget_config_for_deployment", return_value=None), \ + patch.object(limiter, "_get_budget_config_for_tag", return_value=None): + + await limiter.async_log_success_event( + kwargs=kwargs, + response_obj=None, + start_time=None, + end_time=None, + ) + + +@pytest.mark.asyncio +async def test_chat_completions_with_explicit_provider_still_works(): + """ + /v1/chat/completions path that already sets custom_llm_provider must + continue to work exactly as before — no regression. + """ + limiter = _make_limiter() + kwargs = { + "model": "claude-haiku-direct", + "litellm_params": { + "model": "anthropic/claude-haiku-4-5-20251001", + "custom_llm_provider": "anthropic", # set by chat/completions route + }, + "standard_logging_object": { + "response_cost": 0.001, + "model_id": "deployment-abc123", + }, + } + + with patch.object(limiter, "_increment_spend_for_key", new_callable=AsyncMock) as mock_increment, \ + patch.object(limiter, "_get_budget_config_for_deployment", return_value=None), \ + patch.object(limiter, "_get_budget_config_for_tag", return_value=None): + + await limiter.async_log_success_event( + kwargs=kwargs, + response_obj=None, + start_time=None, + end_time=None, + ) + + mock_increment.assert_called_once() From 664c1af541c6a0257b25d5ab246ee82f47fbe750 Mon Sep 17 00:00:00 2001 From: jagjeet-singh-23 Date: Sun, 5 Jul 2026 14:51:14 +0530 Subject: [PATCH 2/6] fix: add debug log on provider derivation failure (greptile suggestion) --- litellm/router_strategy/budget_limiter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index 31240cf1401..542a4e1eca5 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -425,7 +425,12 @@ class RouterBudgetLimiting(CustomLogger): model=str(_model), litellm_params=_LiteLLMParamsDictView(_litellm_params if isinstance(_litellm_params, dict) else {}), ) - except Exception: + except Exception as e: + verbose_router_logger.debug( + "RouterBudgetLimiting: could not derive custom_llm_provider from model string %r: %s", + _model, + e, + ) custom_llm_provider = None if custom_llm_provider is None: raise ValueError("custom_llm_provider is required") From 4e58a95e2bab4bde15892038a01af6ef0a58fd27 Mon Sep 17 00:00:00 2001 From: Jagjeet Singh Date: Sun, 5 Jul 2026 14:51:43 +0530 Subject: [PATCH 3/6] Update litellm/router_strategy/budget_limiter.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/router_strategy/budget_limiter.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index 542a4e1eca5..3d879dd7c2c 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -420,6 +420,11 @@ class RouterBudgetLimiting(CustomLogger): if isinstance(_litellm_params, dict) else getattr(_litellm_params, "model", "") or "" ) + try: + _, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=str(_model), + litellm_params=_LiteLLMParamsDictView(_litellm_params if isinstance(_litellm_params, dict) else {}), + ) try: _, custom_llm_provider, _, _ = litellm.get_llm_provider( model=str(_model), @@ -427,7 +432,8 @@ class RouterBudgetLimiting(CustomLogger): ) except Exception as e: verbose_router_logger.debug( - "RouterBudgetLimiting: could not derive custom_llm_provider from model string %r: %s", + "RouterBudgetLimiting: could not derive custom_llm_provider " + "from model string %r: %s", _model, e, ) From b8d20cb956b00b92497fb29b2ebdc739009a7238 Mon Sep 17 00:00:00 2001 From: jagjeet-singh-23 Date: Sun, 5 Jul 2026 15:03:16 +0530 Subject: [PATCH 4/6] test: cover except branch for unresolvable model string in budget limiter --- ...test_budget_limiter_provider_derivation.py | 53 +++++++++++++++---- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/tests/test_litellm/router_strategy/test_budget_limiter_provider_derivation.py b/tests/test_litellm/router_strategy/test_budget_limiter_provider_derivation.py index a01a004c22e..1266dc8b28e 100644 --- a/tests/test_litellm/router_strategy/test_budget_limiter_provider_derivation.py +++ b/tests/test_litellm/router_strategy/test_budget_limiter_provider_derivation.py @@ -46,9 +46,13 @@ async def test_messages_route_no_custom_provider_does_not_raise(): limiter = _make_limiter() kwargs = _kwargs_without_custom_provider("anthropic/claude-haiku-4-5-20251001") - with patch.object(limiter, "_increment_spend_for_key", new_callable=AsyncMock) as mock_increment, \ - patch.object(limiter, "_get_budget_config_for_deployment", return_value=None), \ - patch.object(limiter, "_get_budget_config_for_tag", return_value=None): + with ( + patch.object( + limiter, "_increment_spend_for_key", new_callable=AsyncMock + ) as mock_increment, + patch.object(limiter, "_get_budget_config_for_deployment", return_value=None), + patch.object(limiter, "_get_budget_config_for_tag", return_value=None), + ): # Before the fix this raises: ValueError("custom_llm_provider is required") await limiter.async_log_success_event( @@ -68,9 +72,11 @@ async def test_embeddings_route_no_custom_provider_does_not_raise(): limiter = _make_limiter() kwargs = _kwargs_without_custom_provider("openai/text-embedding-3-small") - with patch.object(limiter, "_increment_spend_for_key", new_callable=AsyncMock), \ - patch.object(limiter, "_get_budget_config_for_deployment", return_value=None), \ - patch.object(limiter, "_get_budget_config_for_tag", return_value=None): + with ( + patch.object(limiter, "_increment_spend_for_key", new_callable=AsyncMock), + patch.object(limiter, "_get_budget_config_for_deployment", return_value=None), + patch.object(limiter, "_get_budget_config_for_tag", return_value=None), + ): await limiter.async_log_success_event( kwargs=kwargs, @@ -99,9 +105,13 @@ async def test_chat_completions_with_explicit_provider_still_works(): }, } - with patch.object(limiter, "_increment_spend_for_key", new_callable=AsyncMock) as mock_increment, \ - patch.object(limiter, "_get_budget_config_for_deployment", return_value=None), \ - patch.object(limiter, "_get_budget_config_for_tag", return_value=None): + with ( + patch.object( + limiter, "_increment_spend_for_key", new_callable=AsyncMock + ) as mock_increment, + patch.object(limiter, "_get_budget_config_for_deployment", return_value=None), + patch.object(limiter, "_get_budget_config_for_tag", return_value=None), + ): await limiter.async_log_success_event( kwargs=kwargs, @@ -111,3 +121,28 @@ async def test_chat_completions_with_explicit_provider_still_works(): ) mock_increment.assert_called_once() + + +@pytest.mark.asyncio +async def test_unresolvable_model_string_raises_value_error_with_debug_log(): + """ + When both the dict lookup and litellm.get_llm_provider() fail + (completely unrecognisable model string), ValueError must still be raised + and the debug log must fire — covering the except branch added in #26701. + """ + limiter = _make_limiter() + kwargs = _kwargs_without_custom_provider("totally-unresolvable-garbage-xyz-123") + + with ( + patch.object(limiter, "_increment_spend_for_key", new_callable=AsyncMock), + patch.object(limiter, "_get_budget_config_for_deployment", return_value=None), + patch.object(limiter, "_get_budget_config_for_tag", return_value=None), + ): + + with pytest.raises(ValueError, match="custom_llm_provider is required"): + await limiter.async_log_success_event( + kwargs=kwargs, + response_obj=None, + start_time=None, + end_time=None, + ) From 2eca48d26a268b9adc4e8409c8e2a909921ba55d Mon Sep 17 00:00:00 2001 From: jagjeet-singh-23 Date: Sun, 5 Jul 2026 15:37:58 +0530 Subject: [PATCH 5/6] fix: also fallback to kwargs[model] when litellm_params missing model key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LoggedLiteLLMParams for /v1/messages and /v1/embeddings does not populate the model field, so litellm_params.get('model') returns ''. This caused get_llm_provider to fail silently and budget enforcement to be skipped entirely — meaning API key holders could exhaust provider budgets via those routes with no cap enforced. Fix: extend the fallback chain to kwargs['model'] and then standard_logging_payload['model'] before giving up. Addresses review feedback from veria-ai on #32180. --- litellm/router_strategy/budget_limiter.py | 15 ++++---- ...test_budget_limiter_provider_derivation.py | 36 +++++++++++++++++++ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index 3d879dd7c2c..b9c98bcb4f9 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -420,11 +420,13 @@ class RouterBudgetLimiting(CustomLogger): if isinstance(_litellm_params, dict) else getattr(_litellm_params, "model", "") or "" ) - try: - _, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=str(_model), - litellm_params=_LiteLLMParamsDictView(_litellm_params if isinstance(_litellm_params, dict) else {}), - ) + # litellm_params may not carry `model` for /v1/messages and + # /v1/embeddings routes (LoggedLiteLLMParams gap). Fall back to + # the top-level model field and then the standard logging payload + # so budget enforcement is not silently skipped. + # See: https://github.com/BerriAI/litellm/issues/26701 + if not _model: + _model = kwargs.get("model", "") or standard_logging_payload.get("model", "") or "" try: _, custom_llm_provider, _, _ = litellm.get_llm_provider( model=str(_model), @@ -432,8 +434,7 @@ class RouterBudgetLimiting(CustomLogger): ) except Exception as e: verbose_router_logger.debug( - "RouterBudgetLimiting: could not derive custom_llm_provider " - "from model string %r: %s", + "RouterBudgetLimiting: could not derive custom_llm_provider from model string %r: %s", _model, e, ) diff --git a/tests/test_litellm/router_strategy/test_budget_limiter_provider_derivation.py b/tests/test_litellm/router_strategy/test_budget_limiter_provider_derivation.py index 1266dc8b28e..1136e0bceea 100644 --- a/tests/test_litellm/router_strategy/test_budget_limiter_provider_derivation.py +++ b/tests/test_litellm/router_strategy/test_budget_limiter_provider_derivation.py @@ -146,3 +146,39 @@ async def test_unresolvable_model_string_raises_value_error_with_debug_log(): start_time=None, end_time=None, ) + + +@pytest.mark.asyncio +async def test_provider_derived_from_top_level_model_when_litellm_params_missing_model(): + """ + Regression for veria-ai review comment on #32180: + LoggedLiteLLMParams for /v1/messages does not include `model`, so + litellm_params.get("model") returns "". Must fall back to kwargs["model"] + so that provider spend is actually incremented (budget enforced). + """ + limiter = _make_limiter() + + kwargs = { + "model": "anthropic/claude-haiku-4-5-20251001", # top-level, always present + "litellm_params": { + # model key absent — this is the LoggedLiteLLMParams gap + }, + "standard_logging_object": { + "response_cost": 0.001, + "model_id": "deployment-abc123", + }, + } + + with patch.object(limiter, "_increment_spend_for_key", new_callable=AsyncMock) as mock_increment, \ + patch.object(limiter, "_get_budget_config_for_deployment", return_value=None), \ + patch.object(limiter, "_get_budget_config_for_tag", return_value=None): + + await limiter.async_log_success_event( + kwargs=kwargs, + response_obj=None, + start_time=None, + end_time=None, + ) + + # Budget spend MUST be incremented — if it isn't, budget enforcement is broken + mock_increment.assert_called_once() From 242e96c0eee0a26e16d459525ad8e66a630a23b5 Mon Sep 17 00:00:00 2001 From: jagjeet-singh-23 Date: Sun, 5 Jul 2026 16:13:22 +0530 Subject: [PATCH 6/6] fix: suppress BLE001 on intentional broad exception catch in provider derivation --- litellm/router_strategy/budget_limiter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index b9c98bcb4f9..adc216e87dc 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -432,7 +432,7 @@ class RouterBudgetLimiting(CustomLogger): model=str(_model), litellm_params=_LiteLLMParamsDictView(_litellm_params if isinstance(_litellm_params, dict) else {}), ) - except Exception as e: + except Exception as e: # noqa: BLE001 verbose_router_logger.debug( "RouterBudgetLimiting: could not derive custom_llm_provider from model string %r: %s", _model,