fix(proxy): report the dearest deployment behind a wildcard pattern

A pattern can front several deployments and routing picks among them, but the
listing read only the first match, so `openai/*` deployments priced 9e-06 and
4e-05 advertised 9e-06 while /model_group/info reported 4e-05.

get_wildcard_listing_price becomes get_wildcard_listing_prices and returns one
record per matched deployment. Each then resolves its own effective price and the
group reports the highest, the same aggregation an indexed group already gets, so
a matched deployment configuring nothing is billed at its catalog rate and can be
the dearest member: a 1e-08 override beside an uncustomised sibling now reports
the catalog's 1.5e-07 rather than 1e-08.

The guards are unchanged: only names the model-name index misses reach the
pattern router, and only when some wildcard deployment configures a price.

Also drops _deployment_has_configured_price, inlined into its only caller, and
covers _deployment_listing_price directly: tests/code_coverage_tests/
router_code_coverage.py counts only calls made by name from a test file whose
name contains "router".
This commit is contained in:
ajitsharmas2007 2026-09-21 14:22:51 +05:30
parent 215b458358
commit d8d9018552
4 changed files with 142 additions and 48 deletions

View file

@ -8429,22 +8429,24 @@ def _listing_deployment_prices(
"""The per-deployment prices behind a listed name, one entry per deployment.
A name the router indexes carries its deployments' own records. A wildcard-expanded
name has none, so its deployment's override lives on the pattern instead and is asked
for there; the record is keyed to None so an absent half still falls back to the
catalog entry the listing already resolved.
name has none, so the overrides live on the patterns it expands from and are asked for
there, one record per matched deployment because a request can route to any of them.
Those records are keyed to None so a deployment that configures nothing, or configures
only one half, still falls back to the catalog entry the listing already resolved.
"""
if listing_info is not None:
return listing_info.deployment_prices
wildcard_price: Final = llm_router.get_wildcard_listing_price(lookup_model) if llm_router is not None else None
if wildcard_price is None:
wildcard_prices: Final = llm_router.get_wildcard_listing_prices(lookup_model) if llm_router is not None else ()
if not wildcard_prices:
return (DeploymentListingPrice(cost_map_key=None, input_cost_per_token=None, output_cost_per_token=None),)
return (
return tuple(
DeploymentListingPrice(
cost_map_key=None,
input_cost_per_token=wildcard_price.input_cost_per_token,
output_cost_per_token=wildcard_price.output_cost_per_token,
),
input_cost_per_token=price.input_cost_per_token,
output_cost_per_token=price.output_cost_per_token,
)
for price in wildcard_prices
)

View file

@ -10311,40 +10311,42 @@ class Router:
output_cost_per_token=Router._configured_price(model_info, litellm_params, "output_cost_per_token"),
)
def get_wildcard_listing_price(self, model_name: str) -> DeploymentListingPrice | None:
"""Custom pricing a wildcard deployment configures for a name it expands to.
def get_wildcard_listing_prices(self, model_name: str) -> tuple[DeploymentListingPrice, ...]:
"""One record per wildcard deployment that a name expands from, in match order.
Wildcard-expanded rows are absent from the model-name index, so their deployment's
override is invisible to ``get_model_listing_info`` and the listing would quote the
catalog while the request is billed at the override.
Wildcard-expanded rows are absent from the model-name index, so their deployments'
overrides are invisible to ``get_model_listing_info`` and the listing would quote
the catalog while the request is billed at an override.
Every matched deployment is returned rather than the first, because a request can
route to any of them: the caller resolves each one and reports the dearest, the same
way it treats a group the index does know.
Pattern matching is the expensive path this listing exists to avoid, so it only
runs when a wildcard deployment actually configures a price: with none, the catalog
is already the right answer and nothing here can change it.
"""
if not any(
Router._deployment_has_configured_price(deployment)
for deployments in self.pattern_router.patterns.values()
for deployment in deployments
price.input_cost_per_token is not None or price.output_cost_per_token is not None
for price in (
Router._deployment_listing_price(
deployment.get("model_info") or MappingProxyType({}),
deployment.get("litellm_params") or MappingProxyType({}),
)
for deployments in self.pattern_router.patterns.values()
for deployment in deployments
)
):
return None
return ()
matched: Final = self.pattern_router.route(model_name)
if not matched:
return None
return Router._deployment_listing_price(
matched[0].get("model_info") or MappingProxyType({}),
matched[0].get("litellm_params") or MappingProxyType({}),
return tuple(
Router._deployment_listing_price(
deployment.get("model_info") or MappingProxyType({}),
deployment.get("litellm_params") or MappingProxyType({}),
)
for deployment in (self.pattern_router.route(model_name) or ())
)
@staticmethod
def _deployment_has_configured_price(deployment: Mapping[str, Any]) -> bool:
price: Final = Router._deployment_listing_price(
deployment.get("model_info") or MappingProxyType({}),
deployment.get("litellm_params") or MappingProxyType({}),
)
return price.input_cost_per_token is not None or price.output_cost_per_token is not None
def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]":
"""
Return (max_input_tokens, max_output_tokens) configured or discovered for a concrete

View file

@ -1374,8 +1374,10 @@ def test_create_model_info_response_uses_wildcard_pricing_for_an_expanded_row():
"""
router = MagicMock()
router.get_model_listing_info.return_value = None
router.get_wildcard_listing_price.return_value = DeploymentListingPrice(
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05
router.get_wildcard_listing_prices.return_value = (
DeploymentListingPrice(
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05
),
)
response = create_model_info_response(
@ -1395,11 +1397,74 @@ def test_create_model_info_response_uses_wildcard_pricing_for_an_expanded_row():
}
def test_create_model_info_response_reports_the_dearest_deployment_behind_one_pattern():
"""A pattern can front several deployments, and routing can pick any of them.
Reading only the first quoted 9e-06 while a sibling billed 4e-05, so the listing
advertised less than the request could cost.
"""
router = MagicMock()
router.get_model_listing_info.return_value = None
router.get_wildcard_listing_prices.return_value = (
DeploymentListingPrice(
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=9e-06, output_cost_per_token=9e-05
),
DeploymentListingPrice(
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=4e-05, output_cost_per_token=8e-05
),
)
response = create_model_info_response(
model_id="openai/gpt-4o-mini",
provider="openai",
llm_router=router,
include_pricing=True,
get_model_info=lambda _model: _fake_model_info(
key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07
),
cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}},
)
assert response["pricing"] == {
"input_cost_per_token": 4e-05,
"output_cost_per_token": 9e-05,
}
def test_create_model_info_response_wildcard_sibling_without_an_override_still_bills_catalog():
"""A matched deployment configuring nothing is billed at the catalog, so it can be the
dearest member even when its sibling carries a very cheap override."""
router = MagicMock()
router.get_model_listing_info.return_value = None
router.get_wildcard_listing_prices.return_value = (
DeploymentListingPrice(
cost_map_key="openai/gpt-4o-mini", input_cost_per_token=1e-08, output_cost_per_token=1e-08
),
DeploymentListingPrice(cost_map_key="openai/gpt-4o-mini", input_cost_per_token=None, output_cost_per_token=None),
)
response = create_model_info_response(
model_id="openai/gpt-4o-mini",
provider="openai",
llm_router=router,
include_pricing=True,
get_model_info=lambda _model: _fake_model_info(
key="gpt-4o-mini", input_cost_per_token=1.5e-07, output_cost_per_token=6e-07
),
cost_map={"gpt-4o-mini": {"input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07}},
)
assert response["pricing"] == {
"input_cost_per_token": 1.5e-07,
"output_cost_per_token": 6e-07,
}
def test_create_model_info_response_falls_back_to_catalog_for_an_unpriced_wildcard_row():
"""An unpriced wildcard changes nothing, so the catalog answer still stands."""
router = MagicMock()
router.get_model_listing_info.return_value = None
router.get_wildcard_listing_price.return_value = None
router.get_wildcard_listing_prices.return_value = ()
response = create_model_info_response(
model_id="openai/gpt-4o-mini",
@ -1480,7 +1545,7 @@ def test_create_model_info_response_pricing_does_not_call_router_group_info():
"""Pricing must not pull /v1/models onto the expensive group-info path (#33721)."""
router = MagicMock()
router.get_model_listing_info.return_value = None
router.get_wildcard_listing_price.return_value = None
router.get_wildcard_listing_prices.return_value = ()
response = create_model_info_response(
model_id="gpt-4o",

View file

@ -9574,6 +9574,23 @@ def test_configured_price_ignores_a_malformed_configured_value():
assert price is None, f"expected malformed and negative values to be dropped, got {price}"
def test_deployment_listing_price_keys_on_base_model_and_prefers_litellm_params():
"""The record a deployment contributes: its catalog key, plus its own configured prices.
base_model names the catalog entry when set, because litellm_params.model can be an
opaque backend name, and litellm_params wins over model_info for a price, matching the
copy the router makes when it builds the deployment's cost-map entry.
"""
price = litellm.Router._deployment_listing_price(
{"base_model": "eu.anthropic.claude-opus-5", "input_cost_per_token": 1e-06},
{"model": "bedrock/some-opaque-arn", "input_cost_per_token": 4e-06},
)
assert price == DeploymentListingPrice(
cost_map_key="eu.anthropic.claude-opus-5", input_cost_per_token=4e-06, output_cost_per_token=None
)
def test_get_model_listing_info_keeps_each_deployment_price_separate():
"""A group's price is only correct once each deployment resolves its own price first.
@ -9606,12 +9623,9 @@ def test_get_model_listing_info_keeps_each_deployment_price_separate():
)
def test_get_wildcard_listing_price_reports_a_priced_pattern_deployment():
"""A wildcard-expanded name is absent from the index, so its override lives on the pattern.
Without this the listing quotes the catalog while the request is billed at the
override.
"""
def test_get_wildcard_listing_prices_reports_every_matched_deployment():
"""A pattern can front several deployments and routing can pick any of them, so the
listing needs all their prices, not the first one's: the caller reports the dearest."""
router = litellm.Router(
model_list=[
{
@ -9622,17 +9636,28 @@ def test_get_wildcard_listing_price_reports_a_priced_pattern_deployment():
"input_cost_per_token": 9e-06,
"output_cost_per_token": 9e-05,
},
}
},
{
"model_name": "openai/*",
"litellm_params": {
"model": "openai/*",
"api_key": "test-key",
"input_cost_per_token": 4e-05,
"output_cost_per_token": 8e-05,
},
},
]
)
price = router.get_wildcard_listing_price("openai/gpt-4o-mini")
prices = router.get_wildcard_listing_prices("openai/gpt-4o-mini")
assert price is not None
assert (price.input_cost_per_token, price.output_cost_per_token) == (9e-06, 9e-05)
assert {(p.input_cost_per_token, p.output_cost_per_token) for p in prices} == {
(9e-06, 9e-05),
(4e-05, 8e-05),
}
def test_get_wildcard_listing_price_skips_pattern_matching_when_no_wildcard_is_priced():
def test_get_wildcard_listing_prices_skips_pattern_matching_when_no_wildcard_is_priced():
"""Pattern matching is the expensive path, so an unpriced wildcard must not trigger it.
With no override the catalog is already the right answer, so there is nothing to find.
@ -9642,7 +9667,7 @@ def test_get_wildcard_listing_price_skips_pattern_matching_when_no_wildcard_is_p
)
with patch.object(router.pattern_router, "route", side_effect=AssertionError("pattern matching ran")) as routed:
assert router.get_wildcard_listing_price("openai/gpt-4o-mini") is None
assert router.get_wildcard_listing_prices("openai/gpt-4o-mini") == ()
routed.assert_not_called()