From afef2a0e2f73dcfb675e73511b346ddad9ad8cef Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 05:07:28 +0000 Subject: [PATCH 1/2] fix(budget_reservation): don't reserve backend tier rates for $0 deployments Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../spend_tracking/budget_reservation.py | 19 +++ .../proxy/test_budget_reservation.py | 108 ++++++++++++++++++ 2 files changed, 127 insertions(+) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 7cad3f0a022..f4999d26d6f 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -57,6 +57,9 @@ _COUNTER_ENTITY_TYPES: Final[Mapping[str, str]] = { } +_CUSTOM_RATE_KEYS: Final = ("input_cost_per_token", "output_cost_per_token") + + class _CounterReservationUnavailable(Exception): def __init__( self, @@ -1182,10 +1185,26 @@ def _get_model_cost_infos( return [base, *({**base, "tiered_pricing": table} for table in tiered_tables)] +def _deployment_declares_own_rates(deployment: Mapping[str, Any]) -> bool: + """Whether a deployment's own config replaces the backend model's pricing. + + A deployment that spells out per-token rates and no tier table of its own is + billed at those rates, so the backend model's published tier table must not be + used to estimate it: a deployment priced at 0 would otherwise reserve, and + reject on, budget it can never spend. + """ + sources: Final = (deployment.get("litellm_params") or {}, deployment.get("model_info") or {}) + if any(source.get("tiered_pricing") for source in sources): + return False + return any(source.get(key) is not None for source in sources for key in _CUSTOM_RATE_KEYS) + + def _deployment_tiered_pricing_table( deployment: dict[str, Any], llm_router: Router, ) -> list[dict] | None: + if _deployment_declares_own_rates(deployment): + return None model_id: Final = deployment.get("model_info", {}).get("id") backend_model: Final = deployment.get("litellm_params", {}).get("model") if not isinstance(model_id, str) or not isinstance(backend_model, str): diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 38a346e7fb7..9c88daa9990 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -30,6 +30,7 @@ from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessin from litellm.proxy.spend_tracking.budget_reservation import ( TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, _approximate_input_size, + estimate_request_input_cost, estimate_request_max_cost, get_budget_window_start, invalidate_budget_reservation_counters, @@ -1115,6 +1116,113 @@ def test_reservation_uses_most_expensive_deployment_in_group(): assert estimated == pytest.approx(expected_expensive) +@pytest.mark.parametrize( + "deployment_overrides", + [ + {"litellm_params": {"input_cost_per_token": 0, "output_cost_per_token": 0}}, + {"model_info": {"input_cost_per_token": 0, "output_cost_per_token": 0}}, + ], + ids=["litellm_params", "model_info"], +) +def test_free_deployment_of_tiered_model_reserves_nothing(deployment_overrides): + """A deployment priced at 0 on a model whose published entry carries a tier table + must not be estimated against that table. Spend tracking bills such a deployment at + its own rates, so reserving the published tier rate consumed, and rejected requests + against, budget the deployment can never spend.""" + litellm_params = { + "model": "dashscope/qwen-plus-latest", + "api_key": "sk-fake", + **deployment_overrides.get("litellm_params", {}), + } + router = Router( + model_list=[ + { + "model_name": "qwen-free", + "litellm_params": litellm_params, + **({"model_info": deployment_overrides["model_info"]} if "model_info" in deployment_overrides else {}), + } + ] + ) + request_body = { + "model": "qwen-free", + "messages": [{"role": "user", "content": "hello " * 100}], + "max_tokens": 500, + } + + assert ( + estimate_request_max_cost( + request_body=request_body, + route="/chat/completions", + llm_router=router, + ) + == 0.0 + ) + assert ( + estimate_request_input_cost( + request_body=request_body, + route="/chat/completions", + llm_router=router, + ) + == 0.0 + ) + + +def test_priced_deployment_of_tiered_model_still_reserves_tier_rate(): + """The published tier table still governs a deployment that declares no rates of + its own, so the free-deployment carve-out cannot silently disable reservation.""" + router = Router( + model_list=[ + { + "model_name": "qwen-paid", + "litellm_params": {"model": "dashscope/qwen-plus-latest", "api_key": "sk-fake"}, + } + ] + ) + + estimated = estimate_request_max_cost( + request_body={ + "model": "qwen-paid", + "messages": [{"role": "user", "content": "hello " * 100}], + "max_tokens": 500, + }, + route="/chat/completions", + llm_router=router, + ) + + assert estimated is not None and estimated > 0 + + +def test_deployment_declaring_own_tier_table_keeps_it(): + """A deployment that overrides pricing with its own tier table is estimated against + that table, not skipped as if it were unpriced.""" + router = Router( + model_list=[ + { + "model_name": "qwen-own-tiers", + "litellm_params": { + "model": "dashscope/qwen-plus-latest", + "api_key": "sk-fake", + "tiered_pricing": [ + {"range": [0, 1000000], "input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06} + ], + }, + } + ] + ) + + estimated = estimate_request_max_cost( + request_body={ + "model": "qwen-own-tiers", + "messages": [{"role": "user", "content": "hello " * 100}], + "max_tokens": 500, + }, + route="/chat/completions", + llm_router=router, + ) + + assert estimated is not None and estimated > 0 + + @pytest.mark.asyncio async def test_should_clamp_reservation_to_model_ceiling_when_caller_overrequests( spend_counter_state, From d3c8b5d44d2e03411afa9c066fb0cc64fef0bea2 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 05:54:00 +0000 Subject: [PATCH 2/2] fix(budget_reservation): fall back to the model's output rate for input-only tiers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../spend_tracking/budget_reservation.py | 21 +++++++--- .../proxy/test_budget_reservation.py | 42 ++++++++++++++++++- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index f4999d26d6f..87a9206825a 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -1045,6 +1045,21 @@ def _estimate_request_max_cost_for_model( return max(valid_estimates) if valid_estimates else None +_TIER_OUTPUT_RATE_KEYS: Final = ("output_cost_per_token", "output_cost_per_reasoning_token") + + +def _tier_output_rate(tier: Mapping[str, object], model_info: Mapping[str, object]) -> float: + """Output rate to reserve for a request billed at ``tier``. + + A tier table that prices only input falls back to the model's own output rates when + the request is billed, so reserving the tier's missing rate as 0 leaves every + completion under-reserved. The reasoning-token share is unknown before the request + runs, so the higher of the two rates is used either way. + """ + rates: Final = tier if any(key in tier for key in _TIER_OUTPUT_RATE_KEYS) else model_info + return max(_to_float(rates.get(key)) or 0.0 for key in _TIER_OUTPUT_RATE_KEYS) + + def _max_cost_for_cost_info( request_body: dict, route: str, @@ -1079,12 +1094,8 @@ def _max_cost_for_cost_info( if isinstance(tiered_pricing, list) and tiered_pricing: tier: Final = select_tier_for_input(tiered_pricing=tiered_pricing, input_tokens=estimated_input_tokens) if tier is not None: - output_rate = max( - tier_rate(tier, "output_cost_per_token"), - tier_rate(tier, "output_cost_per_reasoning_token"), - ) return (estimated_input_tokens * tier_rate(tier, "input_cost_per_token")) + ( - output_tokens * output_multiplier * output_rate + output_tokens * output_multiplier * _tier_output_rate(tier=tier, model_info=model_info) ) input_cost_per_token: Final = _to_float(model_info.get("input_cost_per_token")) diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 9c88daa9990..d837a7372e2 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -1124,7 +1124,7 @@ def test_reservation_uses_most_expensive_deployment_in_group(): ], ids=["litellm_params", "model_info"], ) -def test_free_deployment_of_tiered_model_reserves_nothing(deployment_overrides): +def test_free_deployment_of_tiered_model_reserves_nothing(deployment_overrides: dict[str, dict[str, int]]): """A deployment priced at 0 on a model whose published entry carries a tier table must not be estimated against that table. Spend tracking bills such a deployment at its own rates, so reserving the published tier rate consumed, and rejected requests @@ -1223,6 +1223,46 @@ def test_deployment_declaring_own_tier_table_keeps_it(): assert estimated is not None and estimated > 0 +def test_input_only_tier_reserves_the_models_own_output_rate(): + """A tier table that prices only input is billed with the model's own output rates, + so reserving the tier's absent output rate as 0 would leave every completion + unreserved and let a budgeted caller run past their limit.""" + output_tokens = 500 + router = Router( + model_list=[ + { + "model_name": "input-tiered", + "litellm_params": { + "model": "dashscope/qwen-plus-latest", + "api_key": "sk-fake", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 5e-06, + "tiered_pricing": [{"range": [0, 32000], "input_cost_per_token": 2e-06}], + }, + } + ] + ) + request_body = { + "model": "input-tiered", + "messages": [{"role": "user", "content": "hello " * 100}], + "max_tokens": output_tokens, + } + + input_cost = estimate_request_input_cost( + request_body=request_body, + route="/chat/completions", + llm_router=router, + ) + estimated = estimate_request_max_cost( + request_body=request_body, + route="/chat/completions", + llm_router=router, + ) + + assert input_cost is not None and input_cost > 0 + assert estimated == pytest.approx(input_cost + (output_tokens * 5e-06)) + + @pytest.mark.asyncio async def test_should_clamp_reservation_to_model_ceiling_when_caller_overrequests( spend_counter_state,