From d6954d57b57f9d4e3cf20a0ecb59e57cc6bd33d8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:57:28 -0700 Subject: [PATCH] fix(proxy): price expanded wildcard /model/info rows as the model their calls route to --- litellm/proxy/auth/model_checks.py | 55 +++++ .../model_management_endpoints.py | 16 +- litellm/proxy/proxy_server.py | 13 +- .../test_model_management_endpoints.py | 22 ++ .../proxy_server/test_routes_model_info.py | 197 ++++++++++++++++++ 5 files changed, 297 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 48884d0fedc..d0c4cf4d9f2 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -7,6 +7,8 @@ import re from collections.abc import Callable, Mapping, Sequence from typing import Final +from pydantic import TypeAdapter + import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor @@ -27,6 +29,8 @@ _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) +_ROUTER_DEPLOYMENT: Final = TypeAdapter(object) + def _check_wildcard_routing(model: str) -> bool: """ @@ -534,6 +538,57 @@ def expand_wildcard_deployments_for_model_info( return expanded +def _litellm_model(deployment: Mapping[str, object]) -> object: + litellm_params: Final = deployment.get("litellm_params") + return litellm_params.get("model") if is_str_keyed_mapping(litellm_params) else None + + +def _router_deployment(llm_router: Router | None, model_id: object) -> Mapping[str, object] | None: + if llm_router is None or not isinstance(model_id, str): + return None + deployment: Final = _ROUTER_DEPLOYMENT.validate_python( + llm_router.get_model_info(id=model_id) # pyright: ignore[reportUnknownMemberType] # narrowed below + ) + return deployment if is_str_keyed_mapping(deployment) else None + + +def wildcard_filled_model(model_name_pattern: str, litellm_model: str, listed_name: str) -> str | None: + listed_name_match: Final = re.match(PatternMatchRouter().pattern_to_regex(model_name_pattern), listed_name) + return ( + None + if listed_name_match is None + else PatternMatchRouter.set_deployment_model_name( + matched_pattern=listed_name_match, litellm_deployment_litellm_model=litellm_model + ) + ) + + +def _deployment_routed_model(source: Mapping[str, object], listed_name: str) -> str | None: + source_pattern: Final = source.get("model_name") + source_model: Final = _litellm_model(source) + return ( + wildcard_filled_model(source_pattern, source_model, listed_name) + if isinstance(source_pattern, str) and isinstance(source_model, str) + else None + ) + + +def wildcard_routed_model(row: Mapping[str, object], llm_router: Router | None) -> str | None: + source: Final = _router_deployment(llm_router, _model_info_field(row, "id")) + listed_name: Final = row.get("model_name") + if llm_router is None or source is None or not isinstance(listed_name, str): + return None + if _litellm_model(row) == _litellm_model(source) or listed_name in llm_router.model_names: + return None + routed_model: Final = _deployment_routed_model(source, listed_name) + return ( + routed_model + if routed_model is not None + and routed_model in llm_router.wildcard_routed_models(listed_name, _deployment_team_id(row)) + else None + ) + + def _get_wildcard_models( unique_models: list[str], return_wildcard_routes: bool | None = False, diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index e7ad7d04070..4762a78f7a6 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -61,6 +61,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY +from litellm.proxy.auth.model_checks import wildcard_filled_model from litellm.proxy.auth.team_grants import team_model_aliases from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.config_sync_pubsub import ( @@ -1264,9 +1265,18 @@ def _ptu_priced_deployment(model_params: Deployment) -> Deployment: ) -def _cost_map_entry(db_model: Deployment, incoming_model_info: Mapping[str, object]) -> Mapping[str, object]: +def _live_cost_map_model(db_model: Deployment, listed_name: str | None) -> str | None: + stored_model: Final = _decrypted_model(db_model.litellm_params.model) + if stored_model is None or listed_name is None: + return stored_model + return wildcard_filled_model(db_model.model_name, stored_model, listed_name) or stored_model + + +def _cost_map_entry( + db_model: Deployment, incoming_model_info: Mapping[str, object], listed_name: str | None +) -> Mapping[str, object]: base_model: Final = incoming_model_info.get("base_model") - lookup: Final = base_model if isinstance(base_model, str) else _decrypted_model(db_model.litellm_params.model) + lookup: Final = base_model if isinstance(base_model, str) else _live_cost_map_model(db_model, listed_name) if lookup is None: return MappingProxyType({}) with suppress(Exception): @@ -1325,7 +1335,7 @@ def update_db_model( incoming_model_info: Final = updated_patch.model_info.model_dump(exclude_none=True) echoed_fields: Final = echoed_cost_map_fields( incoming_model_info, - _cost_map_entry(db_model, incoming_model_info), + _cost_map_entry(db_model, incoming_model_info, updated_patch.model_name), _loaded_catalog_entry(incoming_model_info, loaded_catalog), ) stored_model_info_fields: Final[Mapping[str, object]] = without_server_derived_pricing(incoming_model_info) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 091e05610ba..df5fae012df 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -441,6 +441,7 @@ from litellm.proxy.auth.model_checks import ( get_key_models, get_mcp_server_ids, get_team_models, + wildcard_routed_model, ) from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # legacy module exports @@ -16083,12 +16084,18 @@ def _enrich_model_info_with_litellm_data( # read litellm model_prices_and_context_window.json to get the following: # input_cost_per_token, output_cost_per_token, max_tokens - litellm_model_info = get_litellm_model_info(model=model) + routed_model: Final = wildcard_routed_model(model, llm_router) + priced_model: Final = ( + model + if routed_model is None + else {**model, "litellm_params": {**model["litellm_params"], "model": routed_model}} + ) + litellm_model_info = get_litellm_model_info(model=priced_model) # 2nd pass on the model, try seeing if we can find model in litellm model_cost map if litellm_model_info == {}: # use litellm_param model_name to get model_info - litellm_params = model.get("litellm_params", {}) + litellm_params = priced_model.get("litellm_params", {}) litellm_model = litellm_params.get("model", None) try: litellm_model_info = litellm.get_model_info(model=litellm_model) @@ -16097,7 +16104,7 @@ def _enrich_model_info_with_litellm_data( # 3rd pass on the model, try seeing if we can find model but without the "/" in model cost map if litellm_model_info == {}: # use litellm_param model_name to get model_info - litellm_params = model.get("litellm_params", {}) + litellm_params = priced_model.get("litellm_params", {}) litellm_model = litellm_params.get("model", None) if litellm_model: split_model: Final = litellm_model.split("/") diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index b892dc97e5c..9d81f7b8b12 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -4431,6 +4431,28 @@ class TestModelInfoCostMapEchoFilter: "cost-map metadata must not be persisted from an unchanged /model/info echo" ) + def test_echo_of_an_expanded_wildcard_row_is_not_persisted_on_the_wildcard(self): + import litellm + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("openai/gpt-5.6") + echo = {**entry, "id": "dep-echo-wildcard", "db_model": True, "access_groups": ["prod"]} + db_model = Deployment( + model_name="my-openai/*", + litellm_params=LiteLLM_Params(model="openai/*"), + model_info=ModelInfo(id="dep-echo-wildcard"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_name="my-openai/gpt-5.6", model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert info["access_groups"] == ["prod"] + assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto the whole wildcard" + def test_an_edited_value_survives_the_echo_filter(self): import litellm from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 9993361d930..20721ca24cf 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -578,6 +578,203 @@ async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monke assert [model["model_name"] for model in result] == ["openai/gpt-4o"] +def test_model_info_prices_expanded_wildcard_rows_as_the_model_their_calls_route_to( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """A catch-all over Fireworks lists ``acme-chat``, which also names a DeepSeek entry, and a + prefixed OpenAI wildcard lists ``my-openai/gpt-acme``, which names no entry at all. Each row must + show the provider and price its calls are billed at.""" + from litellm.proxy.auth import model_checks + + catalog: Final = { + "fireworks_ai/accounts/fireworks/models/acme-chat": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "acme-chat": { + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9e-06, + "litellm_provider": "deepseek", + "mode": "chat", + }, + "gpt-acme": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "openai", + "mode": "chat", + }, + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) + _invalidate_model_cost_lowercase_map() + monkeypatch.setattr( + model_checks, + "get_provider_models", + lambda provider, litellm_params=None: { + "fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-chat"], + "openai": ["gpt-acme"], + }.get(provider, []), + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "*", + "litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/*", "api_key": "x"}, + "model_info": {"id": "fireworks-wildcard"}, + }, + { + "model_name": "my-openai/*", + "litellm_params": {"model": "openai/*", "api_key": "x"}, + "model_info": {"id": "openai-wildcard"}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + + with auth_as(): + response: Final = client.get("/model/info") + + assert response.status_code == 200, response.text + listed: Final = { + row["model_name"]: ( + row["litellm_params"]["model"], + row["model_info"].get("litellm_provider"), + row["model_info"].get("input_cost_per_token"), + row["model_info"].get("output_cost_per_token"), + ) + for row in response.json()["data"] + } + assert listed == { + "acme-chat": ("acme-chat", "fireworks_ai", 1e-07, 2e-07), + "my-openai/gpt-acme": ("my-openai/gpt-acme", "openai", 3e-07, 4e-07), + } + calls: Final = { + model_name: router.completion( + model=model_name, messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + for model_name in listed + } + assert {model_name: call._hidden_params["response_cost"] for model_name, call in calls.items()} == pytest.approx( + { + model_name: listed[model_name][2] * call.usage.prompt_tokens + + listed[model_name][3] * call.usage.completion_tokens + for model_name, call in calls.items() + } + ) + _invalidate_model_cost_lowercase_map() + + +def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """``acme-lite`` has a deployment of its own and ``my-openai/gpt-acme-*`` is a narrower wildcard, + so calls to those names never reach the catch-all or ``my-openai/*``, which still list them. Such a + row may go unpriced, but any price a row shows must be the one its calls are billed at.""" + from litellm.proxy.auth import model_checks + + catalog: Final = { + "fireworks_ai/accounts/fireworks/models/acme-lite": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "deepseek/acme-lite": { + "input_cost_per_token": 7e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "deepseek", + "mode": "chat", + }, + "gpt-acme-lite": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "openai", + "mode": "chat", + }, + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) + _invalidate_model_cost_lowercase_map() + monkeypatch.setattr( + model_checks, + "get_provider_models", + lambda provider, litellm_params=None: { + "fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-lite"], + "openai": ["gpt-acme-lite"], + }.get(provider, []), + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "*", + "litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/*", "api_key": "x"}, + "model_info": {"id": "fireworks-wildcard"}, + }, + { + "model_name": "acme-lite", + "litellm_params": {"model": "deepseek/acme-lite", "api_key": "x"}, + "model_info": {"id": "deepseek-acme-lite"}, + }, + { + "model_name": "my-openai/*", + "litellm_params": {"model": "openai/*", "api_key": "x"}, + "model_info": {"id": "openai-wildcard"}, + }, + { + "model_name": "my-openai/gpt-acme-*", + "litellm_params": {"model": "deepseek/acme-*", "api_key": "x"}, + "model_info": {"id": "deepseek-wildcard"}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + + with auth_as(): + response: Final = client.get("/model/info") + + assert response.status_code == 200, response.text + listed: Final = { + (row["model_info"]["id"], row["model_name"]): ( + row["model_info"].get("input_cost_per_token"), + row["model_info"].get("output_cost_per_token"), + ) + for row in response.json()["data"] + if "*" not in row["model_name"] + } + assert sorted(listed) == [ + ("deepseek-acme-lite", "acme-lite"), + ("fireworks-wildcard", "acme-lite"), + ("openai-wildcard", "my-openai/gpt-acme-lite"), + ] + assert listed[("deepseek-acme-lite", "acme-lite")] == (7e-07, 8e-07) + calls: Final = { + model_name: router.completion( + model=model_name, messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + for _, model_name in listed + } + priced_rows: Final = {key: prices for key, prices in listed.items() if prices[0] is not None} + assert { + key: calls[key[1]]._hidden_params["response_cost"] for key in priced_rows + } == pytest.approx( + { + key: input_cost * calls[key[1]].usage.prompt_tokens + output_cost * calls[key[1]].usage.completion_tokens + for key, (input_cost, output_cost) in priced_rows.items() + } + ) + _invalidate_model_cost_lowercase_map() + + # --------------------------------------------------------------------------- # GET /model/info — team BYOK scoping (issue #30983) # ---------------------------------------------------------------------------