diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index b10420e9d50..80ea126343c 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -8429,22 +8429,24 @@ def _listing_deployment_prices( """The per-deployment prices behind a listed name, one entry per deployment. A name the router indexes carries its deployments' own records. A wildcard-expanded - name has none, so its deployment's override lives on the pattern instead and is asked - for there; the record is keyed to None so an absent half still falls back to the - catalog entry the listing already resolved. + name has none, so the overrides live on the patterns it expands from and are asked for + there, one record per matched deployment because a request can route to any of them. + Those records are keyed to None so a deployment that configures nothing, or configures + only one half, still falls back to the catalog entry the listing already resolved. """ if listing_info is not None: return listing_info.deployment_prices - wildcard_price: Final = llm_router.get_wildcard_listing_price(lookup_model) if llm_router is not None else None - if wildcard_price is None: + wildcard_prices: Final = llm_router.get_wildcard_listing_prices(lookup_model) if llm_router is not None else () + if not wildcard_prices: return (DeploymentListingPrice(cost_map_key=None, input_cost_per_token=None, output_cost_per_token=None),) - return ( + return tuple( DeploymentListingPrice( cost_map_key=None, - input_cost_per_token=wildcard_price.input_cost_per_token, - output_cost_per_token=wildcard_price.output_cost_per_token, - ), + input_cost_per_token=price.input_cost_per_token, + output_cost_per_token=price.output_cost_per_token, + ) + for price in wildcard_prices ) diff --git a/litellm/router.py b/litellm/router.py index 866098baf2f..bca31b6427f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10311,40 +10311,42 @@ class Router: output_cost_per_token=Router._configured_price(model_info, litellm_params, "output_cost_per_token"), ) - def get_wildcard_listing_price(self, model_name: str) -> DeploymentListingPrice | None: - """Custom pricing a wildcard deployment configures for a name it expands to. + def get_wildcard_listing_prices(self, model_name: str) -> tuple[DeploymentListingPrice, ...]: + """One record per wildcard deployment that a name expands from, in match order. - Wildcard-expanded rows are absent from the model-name index, so their deployment's - override is invisible to ``get_model_listing_info`` and the listing would quote the - catalog while the request is billed at the override. + Wildcard-expanded rows are absent from the model-name index, so their deployments' + overrides are invisible to ``get_model_listing_info`` and the listing would quote + the catalog while the request is billed at an override. + + Every matched deployment is returned rather than the first, because a request can + route to any of them: the caller resolves each one and reports the dearest, the same + way it treats a group the index does know. Pattern matching is the expensive path this listing exists to avoid, so it only runs when a wildcard deployment actually configures a price: with none, the catalog is already the right answer and nothing here can change it. """ if not any( - Router._deployment_has_configured_price(deployment) - for deployments in self.pattern_router.patterns.values() - for deployment in deployments + price.input_cost_per_token is not None or price.output_cost_per_token is not None + for price in ( + Router._deployment_listing_price( + deployment.get("model_info") or MappingProxyType({}), + deployment.get("litellm_params") or MappingProxyType({}), + ) + for deployments in self.pattern_router.patterns.values() + for deployment in deployments + ) ): - return None + return () - matched: Final = self.pattern_router.route(model_name) - if not matched: - return None - return Router._deployment_listing_price( - matched[0].get("model_info") or MappingProxyType({}), - matched[0].get("litellm_params") or MappingProxyType({}), + return tuple( + Router._deployment_listing_price( + deployment.get("model_info") or MappingProxyType({}), + deployment.get("litellm_params") or MappingProxyType({}), + ) + for deployment in (self.pattern_router.route(model_name) or ()) ) - @staticmethod - def _deployment_has_configured_price(deployment: Mapping[str, Any]) -> bool: - price: Final = Router._deployment_listing_price( - deployment.get("model_info") or MappingProxyType({}), - deployment.get("litellm_params") or MappingProxyType({}), - ) - return price.input_cost_per_token is not None or price.output_cost_per_token is not None - def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": """ Return (max_input_tokens, max_output_tokens) configured or discovered for a concrete diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 0fcb8182962..dfab3d9cfac 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -1374,8 +1374,10 @@ def test_create_model_info_response_uses_wildcard_pricing_for_an_expanded_row(): """ router = MagicMock() router.get_model_listing_info.return_value = None - router.get_wildcard_listing_price.return_value = DeploymentListingPrice( - cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05 + router.get_wildcard_listing_prices.return_value = ( + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05 + ), ) response = create_model_info_response( @@ -1395,11 +1397,74 @@ def test_create_model_info_response_uses_wildcard_pricing_for_an_expanded_row(): } +def test_create_model_info_response_reports_the_dearest_deployment_behind_one_pattern(): + """A pattern can front several deployments, and routing can pick any of them. + + Reading only the first quoted 9e-06 while a sibling billed 4e-05, so the listing + advertised less than the request could cost. + """ + router = MagicMock() + router.get_model_listing_info.return_value = None + router.get_wildcard_listing_prices.return_value = ( + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05 + ), + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=4e-05, output_cost_per_token=8e-05 + ), + ) + + response = create_model_info_response( + model_id="openai/gpt-4o-mini", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07 + ), + cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 4e-05, + "output_cost_per_token": 9e-05, + } + + +def test_create_model_info_response_wildcard_sibling_without_an_override_still_bills_catalog(): + """A matched deployment configuring nothing is billed at the catalog, so it can be the + dearest member even when its sibling carries a very cheap override.""" + router = MagicMock() + router.get_model_listing_info.return_value = None + router.get_wildcard_listing_prices.return_value = ( + DeploymentListingPrice( + cost_map_key="openai/gpt-4o-mini", input_cost_per_token=1e-08, output_cost_per_token=1e-08 + ), + DeploymentListingPrice(cost_map_key="openai/gpt-4o-mini", input_cost_per_token=None, output_cost_per_token=None), + ) + + response = create_model_info_response( + model_id="openai/gpt-4o-mini", + provider="openai", + llm_router=router, + include_pricing=True, + get_model_info=lambda _model: _fake_model_info( + key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07 + ), + cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}}, + ) + + assert response["pricing"] == { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + } + + def test_create_model_info_response_falls_back_to_catalog_for_an_unpriced_wildcard_row(): """An unpriced wildcard changes nothing, so the catalog answer still stands.""" router = MagicMock() router.get_model_listing_info.return_value = None - router.get_wildcard_listing_price.return_value = None + router.get_wildcard_listing_prices.return_value = () response = create_model_info_response( model_id="openai/gpt-4o-mini", @@ -1480,7 +1545,7 @@ def test_create_model_info_response_pricing_does_not_call_router_group_info(): """Pricing must not pull /v1/models onto the expensive group-info path (#33721).""" router = MagicMock() router.get_model_listing_info.return_value = None - router.get_wildcard_listing_price.return_value = None + router.get_wildcard_listing_prices.return_value = () response = create_model_info_response( model_id="gpt-4o", diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 63b33638460..b2f81c31d56 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -9574,6 +9574,23 @@ def test_configured_price_ignores_a_malformed_configured_value(): assert price is None, f"expected malformed and negative values to be dropped, got {price}" +def test_deployment_listing_price_keys_on_base_model_and_prefers_litellm_params(): + """The record a deployment contributes: its catalog key, plus its own configured prices. + + base_model names the catalog entry when set, because litellm_params.model can be an + opaque backend name, and litellm_params wins over model_info for a price, matching the + copy the router makes when it builds the deployment's cost-map entry. + """ + price = litellm.Router._deployment_listing_price( + {"base_model": "eu.anthropic.claude-opus-5", "input_cost_per_token": 1e-06}, + {"model": "bedrock/some-opaque-arn", "input_cost_per_token": 4e-06}, + ) + + assert price == DeploymentListingPrice( + cost_map_key="eu.anthropic.claude-opus-5", input_cost_per_token=4e-06, output_cost_per_token=None + ) + + def test_get_model_listing_info_keeps_each_deployment_price_separate(): """A group's price is only correct once each deployment resolves its own price first. @@ -9606,12 +9623,9 @@ def test_get_model_listing_info_keeps_each_deployment_price_separate(): ) -def test_get_wildcard_listing_price_reports_a_priced_pattern_deployment(): - """A wildcard-expanded name is absent from the index, so its override lives on the pattern. - - Without this the listing quotes the catalog while the request is billed at the - override. - """ +def test_get_wildcard_listing_prices_reports_every_matched_deployment(): + """A pattern can front several deployments and routing can pick any of them, so the + listing needs all their prices, not the first one's: the caller reports the dearest.""" router = litellm.Router( model_list=[ { @@ -9622,17 +9636,28 @@ def test_get_wildcard_listing_price_reports_a_priced_pattern_deployment(): "input_cost_per_token": 9e-06, "output_cost_per_token": 9e-05, }, - } + }, + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "test-key", + "input_cost_per_token": 4e-05, + "output_cost_per_token": 8e-05, + }, + }, ] ) - price = router.get_wildcard_listing_price("openai/gpt-4o-mini") + prices = router.get_wildcard_listing_prices("openai/gpt-4o-mini") - assert price is not None - assert (price.input_cost_per_token, price.output_cost_per_token) == (9e-06, 9e-05) + assert {(p.input_cost_per_token, p.output_cost_per_token) for p in prices} == { + (9e-06, 9e-05), + (4e-05, 8e-05), + } -def test_get_wildcard_listing_price_skips_pattern_matching_when_no_wildcard_is_priced(): +def test_get_wildcard_listing_prices_skips_pattern_matching_when_no_wildcard_is_priced(): """Pattern matching is the expensive path, so an unpriced wildcard must not trigger it. With no override the catalog is already the right answer, so there is nothing to find. @@ -9642,7 +9667,7 @@ def test_get_wildcard_listing_price_skips_pattern_matching_when_no_wildcard_is_p ) with patch.object(router.pattern_router, "route", side_effect=AssertionError("pattern matching ran")) as routed: - assert router.get_wildcard_listing_price("openai/gpt-4o-mini") is None + assert router.get_wildcard_listing_prices("openai/gpt-4o-mini") == () routed.assert_not_called()