mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(proxy): report the dearest deployment behind a wildcard pattern
A pattern can front several deployments and routing picks among them, but the listing read only the first match, so `openai/*` deployments priced 9e-06 and 4e-05 advertised 9e-06 while /model_group/info reported 4e-05. get_wildcard_listing_price becomes get_wildcard_listing_prices and returns one record per matched deployment. Each then resolves its own effective price and the group reports the highest, the same aggregation an indexed group already gets, so a matched deployment configuring nothing is billed at its catalog rate and can be the dearest member: a 1e-08 override beside an uncustomised sibling now reports the catalog's 1.5e-07 rather than 1e-08. The guards are unchanged: only names the model-name index misses reach the pattern router, and only when some wildcard deployment configures a price. Also drops _deployment_has_configured_price, inlined into its only caller, and covers _deployment_listing_price directly: tests/code_coverage_tests/ router_code_coverage.py counts only calls made by name from a test file whose name contains "router".
This commit is contained in:
parent
215b458358
commit
d8d9018552
4 changed files with 142 additions and 48 deletions
|
|
@ -8429,22 +8429,24 @@ def _listing_deployment_prices(
|
|||
"""The per-deployment prices behind a listed name, one entry per deployment.
|
||||
|
||||
A name the router indexes carries its deployments' own records. A wildcard-expanded
|
||||
name has none, so its deployment's override lives on the pattern instead and is asked
|
||||
for there; the record is keyed to None so an absent half still falls back to the
|
||||
catalog entry the listing already resolved.
|
||||
name has none, so the overrides live on the patterns it expands from and are asked for
|
||||
there, one record per matched deployment because a request can route to any of them.
|
||||
Those records are keyed to None so a deployment that configures nothing, or configures
|
||||
only one half, still falls back to the catalog entry the listing already resolved.
|
||||
"""
|
||||
if listing_info is not None:
|
||||
return listing_info.deployment_prices
|
||||
|
||||
wildcard_price: Final = llm_router.get_wildcard_listing_price(lookup_model) if llm_router is not None else None
|
||||
if wildcard_price is None:
|
||||
wildcard_prices: Final = llm_router.get_wildcard_listing_prices(lookup_model) if llm_router is not None else ()
|
||||
if not wildcard_prices:
|
||||
return (DeploymentListingPrice(cost_map_key=None, input_cost_per_token=None, output_cost_per_token=None),)
|
||||
return (
|
||||
return tuple(
|
||||
DeploymentListingPrice(
|
||||
cost_map_key=None,
|
||||
input_cost_per_token=wildcard_price.input_cost_per_token,
|
||||
output_cost_per_token=wildcard_price.output_cost_per_token,
|
||||
),
|
||||
input_cost_per_token=price.input_cost_per_token,
|
||||
output_cost_per_token=price.output_cost_per_token,
|
||||
)
|
||||
for price in wildcard_prices
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10311,40 +10311,42 @@ class Router:
|
|||
output_cost_per_token=Router._configured_price(model_info, litellm_params, "output_cost_per_token"),
|
||||
)
|
||||
|
||||
def get_wildcard_listing_price(self, model_name: str) -> DeploymentListingPrice | None:
|
||||
"""Custom pricing a wildcard deployment configures for a name it expands to.
|
||||
def get_wildcard_listing_prices(self, model_name: str) -> tuple[DeploymentListingPrice, ...]:
|
||||
"""One record per wildcard deployment that a name expands from, in match order.
|
||||
|
||||
Wildcard-expanded rows are absent from the model-name index, so their deployment's
|
||||
override is invisible to ``get_model_listing_info`` and the listing would quote the
|
||||
catalog while the request is billed at the override.
|
||||
Wildcard-expanded rows are absent from the model-name index, so their deployments'
|
||||
overrides are invisible to ``get_model_listing_info`` and the listing would quote
|
||||
the catalog while the request is billed at an override.
|
||||
|
||||
Every matched deployment is returned rather than the first, because a request can
|
||||
route to any of them: the caller resolves each one and reports the dearest, the same
|
||||
way it treats a group the index does know.
|
||||
|
||||
Pattern matching is the expensive path this listing exists to avoid, so it only
|
||||
runs when a wildcard deployment actually configures a price: with none, the catalog
|
||||
is already the right answer and nothing here can change it.
|
||||
"""
|
||||
if not any(
|
||||
Router._deployment_has_configured_price(deployment)
|
||||
for deployments in self.pattern_router.patterns.values()
|
||||
for deployment in deployments
|
||||
price.input_cost_per_token is not None or price.output_cost_per_token is not None
|
||||
for price in (
|
||||
Router._deployment_listing_price(
|
||||
deployment.get("model_info") or MappingProxyType({}),
|
||||
deployment.get("litellm_params") or MappingProxyType({}),
|
||||
)
|
||||
for deployments in self.pattern_router.patterns.values()
|
||||
for deployment in deployments
|
||||
)
|
||||
):
|
||||
return None
|
||||
return ()
|
||||
|
||||
matched: Final = self.pattern_router.route(model_name)
|
||||
if not matched:
|
||||
return None
|
||||
return Router._deployment_listing_price(
|
||||
matched[0].get("model_info") or MappingProxyType({}),
|
||||
matched[0].get("litellm_params") or MappingProxyType({}),
|
||||
return tuple(
|
||||
Router._deployment_listing_price(
|
||||
deployment.get("model_info") or MappingProxyType({}),
|
||||
deployment.get("litellm_params") or MappingProxyType({}),
|
||||
)
|
||||
for deployment in (self.pattern_router.route(model_name) or ())
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _deployment_has_configured_price(deployment: Mapping[str, Any]) -> bool:
|
||||
price: Final = Router._deployment_listing_price(
|
||||
deployment.get("model_info") or MappingProxyType({}),
|
||||
deployment.get("litellm_params") or MappingProxyType({}),
|
||||
)
|
||||
return price.input_cost_per_token is not None or price.output_cost_per_token is not None
|
||||
|
||||
def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]":
|
||||
"""
|
||||
Return (max_input_tokens, max_output_tokens) configured or discovered for a concrete
|
||||
|
|
|
|||
|
|
@ -1374,8 +1374,10 @@ def test_create_model_info_response_uses_wildcard_pricing_for_an_expanded_row():
|
|||
"""
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = None
|
||||
router.get_wildcard_listing_price.return_value = DeploymentListingPrice(
|
||||
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05
|
||||
router.get_wildcard_listing_prices.return_value = (
|
||||
DeploymentListingPrice(
|
||||
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05
|
||||
),
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
|
|
@ -1395,11 +1397,74 @@ def test_create_model_info_response_uses_wildcard_pricing_for_an_expanded_row():
|
|||
}
|
||||
|
||||
|
||||
def test_create_model_info_response_reports_the_dearest_deployment_behind_one_pattern():
|
||||
"""A pattern can front several deployments, and routing can pick any of them.
|
||||
|
||||
Reading only the first quoted 9e-06 while a sibling billed 4e-05, so the listing
|
||||
advertised less than the request could cost.
|
||||
"""
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = None
|
||||
router.get_wildcard_listing_prices.return_value = (
|
||||
DeploymentListingPrice(
|
||||
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05
|
||||
),
|
||||
DeploymentListingPrice(
|
||||
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=4e-05, output_cost_per_token=8e-05
|
||||
),
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="openai/gpt-4o-mini",
|
||||
provider="openai",
|
||||
llm_router=router,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda _model: _fake_model_info(
|
||||
key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07
|
||||
),
|
||||
cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}},
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": 4e-05,
|
||||
"output_cost_per_token": 9e-05,
|
||||
}
|
||||
|
||||
|
||||
def test_create_model_info_response_wildcard_sibling_without_an_override_still_bills_catalog():
|
||||
"""A matched deployment configuring nothing is billed at the catalog, so it can be the
|
||||
dearest member even when its sibling carries a very cheap override."""
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = None
|
||||
router.get_wildcard_listing_prices.return_value = (
|
||||
DeploymentListingPrice(
|
||||
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=1e-08, output_cost_per_token=1e-08
|
||||
),
|
||||
DeploymentListingPrice(cost_map_key="openai/gpt-4o-mini", input_cost_per_token=None, output_cost_per_token=None),
|
||||
)
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="openai/gpt-4o-mini",
|
||||
provider="openai",
|
||||
llm_router=router,
|
||||
include_pricing=True,
|
||||
get_model_info=lambda _model: _fake_model_info(
|
||||
key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07
|
||||
),
|
||||
cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}},
|
||||
)
|
||||
|
||||
assert response["pricing"] == {
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"output_cost_per_token": 6e-07,
|
||||
}
|
||||
|
||||
|
||||
def test_create_model_info_response_falls_back_to_catalog_for_an_unpriced_wildcard_row():
|
||||
"""An unpriced wildcard changes nothing, so the catalog answer still stands."""
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = None
|
||||
router.get_wildcard_listing_price.return_value = None
|
||||
router.get_wildcard_listing_prices.return_value = ()
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="openai/gpt-4o-mini",
|
||||
|
|
@ -1480,7 +1545,7 @@ def test_create_model_info_response_pricing_does_not_call_router_group_info():
|
|||
"""Pricing must not pull /v1/models onto the expensive group-info path (#33721)."""
|
||||
router = MagicMock()
|
||||
router.get_model_listing_info.return_value = None
|
||||
router.get_wildcard_listing_price.return_value = None
|
||||
router.get_wildcard_listing_prices.return_value = ()
|
||||
|
||||
response = create_model_info_response(
|
||||
model_id="gpt-4o",
|
||||
|
|
|
|||
|
|
@ -9574,6 +9574,23 @@ def test_configured_price_ignores_a_malformed_configured_value():
|
|||
assert price is None, f"expected malformed and negative values to be dropped, got {price}"
|
||||
|
||||
|
||||
def test_deployment_listing_price_keys_on_base_model_and_prefers_litellm_params():
|
||||
"""The record a deployment contributes: its catalog key, plus its own configured prices.
|
||||
|
||||
base_model names the catalog entry when set, because litellm_params.model can be an
|
||||
opaque backend name, and litellm_params wins over model_info for a price, matching the
|
||||
copy the router makes when it builds the deployment's cost-map entry.
|
||||
"""
|
||||
price = litellm.Router._deployment_listing_price(
|
||||
{"base_model": "eu.anthropic.claude-opus-5", "input_cost_per_token": 1e-06},
|
||||
{"model": "bedrock/some-opaque-arn", "input_cost_per_token": 4e-06},
|
||||
)
|
||||
|
||||
assert price == DeploymentListingPrice(
|
||||
cost_map_key="eu.anthropic.claude-opus-5", input_cost_per_token=4e-06, output_cost_per_token=None
|
||||
)
|
||||
|
||||
|
||||
def test_get_model_listing_info_keeps_each_deployment_price_separate():
|
||||
"""A group's price is only correct once each deployment resolves its own price first.
|
||||
|
||||
|
|
@ -9606,12 +9623,9 @@ def test_get_model_listing_info_keeps_each_deployment_price_separate():
|
|||
)
|
||||
|
||||
|
||||
def test_get_wildcard_listing_price_reports_a_priced_pattern_deployment():
|
||||
"""A wildcard-expanded name is absent from the index, so its override lives on the pattern.
|
||||
|
||||
Without this the listing quotes the catalog while the request is billed at the
|
||||
override.
|
||||
"""
|
||||
def test_get_wildcard_listing_prices_reports_every_matched_deployment():
|
||||
"""A pattern can front several deployments and routing can pick any of them, so the
|
||||
listing needs all their prices, not the first one's: the caller reports the dearest."""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
|
|
@ -9622,17 +9636,28 @@ def test_get_wildcard_listing_price_reports_a_priced_pattern_deployment():
|
|||
"input_cost_per_token": 9e-06,
|
||||
"output_cost_per_token": 9e-05,
|
||||
},
|
||||
}
|
||||
},
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": "test-key",
|
||||
"input_cost_per_token": 4e-05,
|
||||
"output_cost_per_token": 8e-05,
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
price = router.get_wildcard_listing_price("openai/gpt-4o-mini")
|
||||
prices = router.get_wildcard_listing_prices("openai/gpt-4o-mini")
|
||||
|
||||
assert price is not None
|
||||
assert (price.input_cost_per_token, price.output_cost_per_token) == (9e-06, 9e-05)
|
||||
assert {(p.input_cost_per_token, p.output_cost_per_token) for p in prices} == {
|
||||
(9e-06, 9e-05),
|
||||
(4e-05, 8e-05),
|
||||
}
|
||||
|
||||
|
||||
def test_get_wildcard_listing_price_skips_pattern_matching_when_no_wildcard_is_priced():
|
||||
def test_get_wildcard_listing_prices_skips_pattern_matching_when_no_wildcard_is_priced():
|
||||
"""Pattern matching is the expensive path, so an unpriced wildcard must not trigger it.
|
||||
|
||||
With no override the catalog is already the right answer, so there is nothing to find.
|
||||
|
|
@ -9642,7 +9667,7 @@ def test_get_wildcard_listing_price_skips_pattern_matching_when_no_wildcard_is_p
|
|||
)
|
||||
|
||||
with patch.object(router.pattern_router, "route", side_effect=AssertionError("pattern matching ran")) as routed:
|
||||
assert router.get_wildcard_listing_price("openai/gpt-4o-mini") is None
|
||||
assert router.get_wildcard_listing_prices("openai/gpt-4o-mini") == ()
|
||||
|
||||
routed.assert_not_called()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue