From b7878751f7868ccdb61cb2b486bc24b9bb0ded37 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 03:58:58 -0700 Subject: [PATCH] fix(proxy): show no wildcard row price when the same-model deployment serving it sets custom rates --- litellm/proxy/auth/model_checks.py | 36 ++++++++++++++----- .../proxy_server/test_routes_model_info.py | 22 +++++++++++- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 71848adc094..9021929777b 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -22,7 +22,7 @@ from litellm.router_api import RouterRegistry from litellm.router_utils.fallback_event_handlers import get_fallback_model_group from litellm.router_utils.pattern_match_deployments import PatternMatchRouter from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params -from litellm.types.utils import LlmProviders +from litellm.types.utils import CustomPricingLiteLLMParams, LlmProviders from litellm.utils import ProviderConfigManager, get_valid_models LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders) @@ -620,25 +620,45 @@ def wildcard_deployment_priced_model( candidates: Final = registry.names( served_name, kind="route_candidates", team_id=deployment.team_id or caller_aliases.team_id ) - if deployment.model_id in candidates or _every_candidate_runs(registry, candidates, routed_name, priced): + wildcard_run: Final = (priced, _pricing_overrides(_router_deployment(registry, deployment.model_id))) + if deployment.model_id in candidates or _every_candidate_runs(registry, candidates, routed_name, wildcard_run): return priced return ROUTED_ELSEWHERE -def _every_candidate_runs(registry: RouterRegistry, candidates: frozenset[str], routed_name: str, model: str) -> bool: +_PRICING_FIELDS: Final = frozenset((*CustomPricingLiteLLMParams.model_fields, "base_model")) + + +def _every_candidate_runs( + registry: RouterRegistry, candidates: frozenset[str], routed_name: str, run: tuple[object, Mapping[str, object]] +) -> bool: return bool(candidates) and all( - _model_run_by(_router_deployment(registry, candidate), routed_name) == model for candidate in candidates + _run_of(_router_deployment(registry, candidate), routed_name) == run for candidate in candidates ) -def _model_run_by(deployment: Mapping[str, object] | None, routed_name: str) -> object: +def _run_of(deployment: Mapping[str, object] | None, routed_name: str) -> tuple[object, Mapping[str, object]] | None: if deployment is None: return None model_name: Final = deployment.get("model_name") litellm_model: Final = _litellm_model(deployment) - if not isinstance(model_name, str) or not isinstance(litellm_model, str): - return litellm_model - return wildcard_listed_priced_model(model_name, litellm_model, routed_name) + model: Final = ( + wildcard_listed_priced_model(model_name, litellm_model, routed_name) + if isinstance(model_name, str) and isinstance(litellm_model, str) + else litellm_model + ) + return (model, _pricing_overrides(deployment)) + + +def _pricing_overrides(deployment: Mapping[str, object] | None) -> Mapping[str, object]: + blobs: Final = () if deployment is None else (deployment.get("model_info"), deployment.get("litellm_params")) + return MappingProxyType(dict(itertools.chain.from_iterable(_pricing_items(blob) for blob in blobs))) + + +def _pricing_items(blob: object) -> tuple[tuple[str, object], ...]: + if not is_str_keyed_mapping(blob): + return () + return tuple((key, value) for key, value in blob.items() if key in _PRICING_FIELDS and value is not None) def _without_deleted_team_targets(caller_aliases: CallerAliases, registry: RouterRegistry) -> CallerAliases: diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 064c4574a23..d98603da30c 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -1136,7 +1136,7 @@ def test_model_info_keeps_the_price_of_a_wildcard_row_another_deployment_of_the_ ) -> None: """A deployment pinned to one model a wildcard lists takes that name's calls but runs the model the wildcard would run, so the wildcard's row keeps the price those calls bill, and saving that row back stores none of it. - A name pinned to another model shows no price.""" + A name pinned to another model, or to the same model at custom rates, shows no price.""" from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment @@ -1158,6 +1158,16 @@ def test_model_info_keeps_the_price_of_a_wildcard_row_another_deployment_of_the_ "litellm_params": {"model": "deepseek/acme-lite", "api_key": "x"}, "model_info": {"id": "pinned-acme-pro"}, }, + { + "model_name": "my-fw/acme-max", + "litellm_params": { + "model": "fireworks_ai/accounts/fireworks/models/acme-max", + "api_key": "x", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 3e-06, + }, + "model_info": {"id": "pinned-acme-max"}, + }, ] ) monkeypatch.setattr(proxy_server, "llm_router", router) @@ -1191,6 +1201,16 @@ def test_model_info_keeps_the_price_of_a_wildcard_row_another_deployment_of_the_ assert stored["access_groups"] == ["prod"] assert set(stored).isdisjoint(set(shown) - set(router.registry().deployment_dict("fireworks-wildcard")["model_info"])) assert rows["my-fw/acme-pro"].get("input_cost_per_token") is None + custom_rate_call: Final = router.completion( + model="my-fw/acme-max", messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + assert custom_rate_call._hidden_params["response_cost"] == pytest.approx( + 2e-06 * custom_rate_call.usage.prompt_tokens + 3e-06 * custom_rate_call.usage.completion_tokens + ) + assert (rows["my-fw/acme-max"].get("input_cost_per_token"), rows["my-fw/acme-max"].get("litellm_provider")) == ( + None, + None, + ) _invalidate_model_cost_lowercase_map()