Merge d3c8b5d44d into 493bca667b

litellm/proxy/spend_tracking/budget_reservation.py

@@ -57,6 +57,9 @@ _COUNTER_ENTITY_TYPES: Final[Mapping[str, str]] = {
 }
 
 
+_CUSTOM_RATE_KEYS: Final = ("input_cost_per_token", "output_cost_per_token")
+
+
 class _CounterReservationUnavailable(Exception):
     def __init__(
         self,
@@ -1042,6 +1045,21 @@ def _estimate_request_max_cost_for_model(
     return max(valid_estimates) if valid_estimates else None
 
 
+_TIER_OUTPUT_RATE_KEYS: Final = ("output_cost_per_token", "output_cost_per_reasoning_token")
+
+
+def _tier_output_rate(tier: Mapping[str, object], model_info: Mapping[str, object]) -> float:
+    """Output rate to reserve for a request billed at ``tier``.
+
+    A tier table that prices only input falls back to the model's own output rates when
+    the request is billed, so reserving the tier's missing rate as 0 leaves every
+    completion under-reserved. The reasoning-token share is unknown before the request
+    runs, so the higher of the two rates is used either way.
+    """
+    rates: Final = tier if any(key in tier for key in _TIER_OUTPUT_RATE_KEYS) else model_info
+    return max(_to_float(rates.get(key)) or 0.0 for key in _TIER_OUTPUT_RATE_KEYS)
+
+
 def _max_cost_for_cost_info(
     request_body: dict,
     route: str,
@@ -1076,12 +1094,8 @@ def _max_cost_for_cost_info(
     if isinstance(tiered_pricing, list) and tiered_pricing:
         tier: Final = select_tier_for_input(tiered_pricing=tiered_pricing, input_tokens=estimated_input_tokens)
         if tier is not None:
-            output_rate = max(
-                tier_rate(tier, "output_cost_per_token"),
-                tier_rate(tier, "output_cost_per_reasoning_token"),
-            )
             return (estimated_input_tokens * tier_rate(tier, "input_cost_per_token")) + (
-                output_tokens * output_multiplier * output_rate
+                output_tokens * output_multiplier * _tier_output_rate(tier=tier, model_info=model_info)
             )
 
     input_cost_per_token: Final = _to_float(model_info.get("input_cost_per_token"))
@@ -1182,10 +1196,26 @@ def _get_model_cost_infos(
     return [base, *({**base, "tiered_pricing": table} for table in tiered_tables)]
 
 
+def _deployment_declares_own_rates(deployment: Mapping[str, Any]) -> bool:
+    """Whether a deployment's own config replaces the backend model's pricing.
+
+    A deployment that spells out per-token rates and no tier table of its own is
+    billed at those rates, so the backend model's published tier table must not be
+    used to estimate it: a deployment priced at 0 would otherwise reserve, and
+    reject on, budget it can never spend.
+    """
+    sources: Final = (deployment.get("litellm_params") or {}, deployment.get("model_info") or {})
+    if any(source.get("tiered_pricing") for source in sources):
+        return False
+    return any(source.get(key) is not None for source in sources for key in _CUSTOM_RATE_KEYS)
+
+
 def _deployment_tiered_pricing_table(
     deployment: dict[str, Any],
     llm_router: Router,
 ) -> list[dict] | None:
+    if _deployment_declares_own_rates(deployment):
+        return None
     model_id: Final = deployment.get("model_info", {}).get("id")
     backend_model: Final = deployment.get("litellm_params", {}).get("model")
     if not isinstance(model_id, str) or not isinstance(backend_model, str):

tests/test_litellm/proxy/test_budget_reservation.py

@@ -30,6 +30,7 @@ from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessin
 from litellm.proxy.spend_tracking.budget_reservation import (
     TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS,
     _approximate_input_size,
+    estimate_request_input_cost,
     estimate_request_max_cost,
     get_budget_window_start,
     invalidate_budget_reservation_counters,
@@ -1115,6 +1116,153 @@ def test_reservation_uses_most_expensive_deployment_in_group():
     assert estimated == pytest.approx(expected_expensive)
 
 
+@pytest.mark.parametrize(
+    "deployment_overrides",
+    [
+        {"litellm_params": {"input_cost_per_token": 0, "output_cost_per_token": 0}},
+        {"model_info": {"input_cost_per_token": 0, "output_cost_per_token": 0}},
+    ],
+    ids=["litellm_params", "model_info"],
+)
+def test_free_deployment_of_tiered_model_reserves_nothing(deployment_overrides: dict[str, dict[str, int]]):
+    """A deployment priced at 0 on a model whose published entry carries a tier table
+    must not be estimated against that table. Spend tracking bills such a deployment at
+    its own rates, so reserving the published tier rate consumed, and rejected requests
+    against, budget the deployment can never spend."""
+    litellm_params = {
+        "model": "dashscope/qwen-plus-latest",
+        "api_key": "sk-fake",
+        **deployment_overrides.get("litellm_params", {}),
+    }
+    router = Router(
+        model_list=[
+            {
+                "model_name": "qwen-free",
+                "litellm_params": litellm_params,
+                **({"model_info": deployment_overrides["model_info"]} if "model_info" in deployment_overrides else {}),
+            }
+        ]
+    )
+    request_body = {
+        "model": "qwen-free",
+        "messages": [{"role": "user", "content": "hello " * 100}],
+        "max_tokens": 500,
+    }
+
+    assert (
+        estimate_request_max_cost(
+            request_body=request_body,
+            route="/chat/completions",
+            llm_router=router,
+        )
+        == 0.0
+    )
+    assert (
+        estimate_request_input_cost(
+            request_body=request_body,
+            route="/chat/completions",
+            llm_router=router,
+        )
+        == 0.0
+    )
+
+
+def test_priced_deployment_of_tiered_model_still_reserves_tier_rate():
+    """The published tier table still governs a deployment that declares no rates of
+    its own, so the free-deployment carve-out cannot silently disable reservation."""
+    router = Router(
+        model_list=[
+            {
+                "model_name": "qwen-paid",
+                "litellm_params": {"model": "dashscope/qwen-plus-latest", "api_key": "sk-fake"},
+            }
+        ]
+    )
+
+    estimated = estimate_request_max_cost(
+        request_body={
+            "model": "qwen-paid",
+            "messages": [{"role": "user", "content": "hello " * 100}],
+            "max_tokens": 500,
+        },
+        route="/chat/completions",
+        llm_router=router,
+    )
+
+    assert estimated is not None and estimated > 0
+
+
+def test_deployment_declaring_own_tier_table_keeps_it():
+    """A deployment that overrides pricing with its own tier table is estimated against
+    that table, not skipped as if it were unpriced."""
+    router = Router(
+        model_list=[
+            {
+                "model_name": "qwen-own-tiers",
+                "litellm_params": {
+                    "model": "dashscope/qwen-plus-latest",
+                    "api_key": "sk-fake",
+                    "tiered_pricing": [
+                        {"range": [0, 1000000], "input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06}
+                    ],
+                },
+            }
+        ]
+    )
+
+    estimated = estimate_request_max_cost(
+        request_body={
+            "model": "qwen-own-tiers",
+            "messages": [{"role": "user", "content": "hello " * 100}],
+            "max_tokens": 500,
+        },
+        route="/chat/completions",
+        llm_router=router,
+    )
+
+    assert estimated is not None and estimated > 0
+
+
+def test_input_only_tier_reserves_the_models_own_output_rate():
+    """A tier table that prices only input is billed with the model's own output rates,
+    so reserving the tier's absent output rate as 0 would leave every completion
+    unreserved and let a budgeted caller run past their limit."""
+    output_tokens = 500
+    router = Router(
+        model_list=[
+            {
+                "model_name": "input-tiered",
+                "litellm_params": {
+                    "model": "dashscope/qwen-plus-latest",
+                    "api_key": "sk-fake",
+                    "input_cost_per_token": 1e-06,
+                    "output_cost_per_token": 5e-06,
+                    "tiered_pricing": [{"range": [0, 32000], "input_cost_per_token": 2e-06}],
+                },
+            }
+        ]
+    )
+    request_body = {
+        "model": "input-tiered",
+        "messages": [{"role": "user", "content": "hello " * 100}],
+        "max_tokens": output_tokens,
+    }
+
+    input_cost = estimate_request_input_cost(
+        request_body=request_body,
+        route="/chat/completions",
+        llm_router=router,
+    )
+    estimated = estimate_request_max_cost(
+        request_body=request_body,
+        route="/chat/completions",
+        llm_router=router,
+    )
+
+    assert input_cost is not None and input_cost > 0
+    assert estimated == pytest.approx(input_cost + (output_tokens * 5e-06))
+
+
 @pytest.mark.asyncio
 async def test_should_clamp_reservation_to_model_ceiling_when_caller_overrequests(
     spend_counter_state,