From d6954d57b57f9d4e3cf20a0ecb59e57cc6bd33d8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:57:28 -0700 Subject: [PATCH 01/12] fix(proxy): price expanded wildcard /model/info rows as the model their calls route to --- litellm/proxy/auth/model_checks.py | 55 +++++ .../model_management_endpoints.py | 16 +- litellm/proxy/proxy_server.py | 13 +- .../test_model_management_endpoints.py | 22 ++ .../proxy_server/test_routes_model_info.py | 197 ++++++++++++++++++ 5 files changed, 297 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 48884d0fedc..d0c4cf4d9f2 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -7,6 +7,8 @@ import re from collections.abc import Callable, Mapping, Sequence from typing import Final +from pydantic import TypeAdapter + import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor @@ -27,6 +29,8 @@ _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) +_ROUTER_DEPLOYMENT: Final = TypeAdapter(object) + def _check_wildcard_routing(model: str) -> bool: """ @@ -534,6 +538,57 @@ def expand_wildcard_deployments_for_model_info( return expanded +def _litellm_model(deployment: Mapping[str, object]) -> object: + litellm_params: Final = deployment.get("litellm_params") + return litellm_params.get("model") if is_str_keyed_mapping(litellm_params) else None + + +def _router_deployment(llm_router: Router | None, model_id: object) -> Mapping[str, object] | None: + if llm_router is None or not isinstance(model_id, str): + return None + deployment: Final = _ROUTER_DEPLOYMENT.validate_python( + llm_router.get_model_info(id=model_id) # pyright: ignore[reportUnknownMemberType] # narrowed below + ) + return deployment if is_str_keyed_mapping(deployment) else None + + +def wildcard_filled_model(model_name_pattern: str, litellm_model: str, listed_name: str) -> str | None: + listed_name_match: Final = re.match(PatternMatchRouter().pattern_to_regex(model_name_pattern), listed_name) + return ( + None + if listed_name_match is None + else PatternMatchRouter.set_deployment_model_name( + matched_pattern=listed_name_match, litellm_deployment_litellm_model=litellm_model + ) + ) + + +def _deployment_routed_model(source: Mapping[str, object], listed_name: str) -> str | None: + source_pattern: Final = source.get("model_name") + source_model: Final = _litellm_model(source) + return ( + wildcard_filled_model(source_pattern, source_model, listed_name) + if isinstance(source_pattern, str) and isinstance(source_model, str) + else None + ) + + +def wildcard_routed_model(row: Mapping[str, object], llm_router: Router | None) -> str | None: + source: Final = _router_deployment(llm_router, _model_info_field(row, "id")) + listed_name: Final = row.get("model_name") + if llm_router is None or source is None or not isinstance(listed_name, str): + return None + if _litellm_model(row) == _litellm_model(source) or listed_name in llm_router.model_names: + return None + routed_model: Final = _deployment_routed_model(source, listed_name) + return ( + routed_model + if routed_model is not None + and routed_model in llm_router.wildcard_routed_models(listed_name, _deployment_team_id(row)) + else None + ) + + def _get_wildcard_models( unique_models: list[str], return_wildcard_routes: bool | None = False, diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index e7ad7d04070..4762a78f7a6 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -61,6 +61,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY +from litellm.proxy.auth.model_checks import wildcard_filled_model from litellm.proxy.auth.team_grants import team_model_aliases from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.config_sync_pubsub import ( @@ -1264,9 +1265,18 @@ def _ptu_priced_deployment(model_params: Deployment) -> Deployment: ) -def _cost_map_entry(db_model: Deployment, incoming_model_info: Mapping[str, object]) -> Mapping[str, object]: +def _live_cost_map_model(db_model: Deployment, listed_name: str | None) -> str | None: + stored_model: Final = _decrypted_model(db_model.litellm_params.model) + if stored_model is None or listed_name is None: + return stored_model + return wildcard_filled_model(db_model.model_name, stored_model, listed_name) or stored_model + + +def _cost_map_entry( + db_model: Deployment, incoming_model_info: Mapping[str, object], listed_name: str | None +) -> Mapping[str, object]: base_model: Final = incoming_model_info.get("base_model") - lookup: Final = base_model if isinstance(base_model, str) else _decrypted_model(db_model.litellm_params.model) + lookup: Final = base_model if isinstance(base_model, str) else _live_cost_map_model(db_model, listed_name) if lookup is None: return MappingProxyType({}) with suppress(Exception): @@ -1325,7 +1335,7 @@ def update_db_model( incoming_model_info: Final = updated_patch.model_info.model_dump(exclude_none=True) echoed_fields: Final = echoed_cost_map_fields( incoming_model_info, - _cost_map_entry(db_model, incoming_model_info), + _cost_map_entry(db_model, incoming_model_info, updated_patch.model_name), _loaded_catalog_entry(incoming_model_info, loaded_catalog), ) stored_model_info_fields: Final[Mapping[str, object]] = without_server_derived_pricing(incoming_model_info) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 091e05610ba..df5fae012df 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -441,6 +441,7 @@ from litellm.proxy.auth.model_checks import ( get_key_models, get_mcp_server_ids, get_team_models, + wildcard_routed_model, ) from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # legacy module exports @@ -16083,12 +16084,18 @@ def _enrich_model_info_with_litellm_data( # read litellm model_prices_and_context_window.json to get the following: # input_cost_per_token, output_cost_per_token, max_tokens - litellm_model_info = get_litellm_model_info(model=model) + routed_model: Final = wildcard_routed_model(model, llm_router) + priced_model: Final = ( + model + if routed_model is None + else {**model, "litellm_params": {**model["litellm_params"], "model": routed_model}} + ) + litellm_model_info = get_litellm_model_info(model=priced_model) # 2nd pass on the model, try seeing if we can find model in litellm model_cost map if litellm_model_info == {}: # use litellm_param model_name to get model_info - litellm_params = model.get("litellm_params", {}) + litellm_params = priced_model.get("litellm_params", {}) litellm_model = litellm_params.get("model", None) try: litellm_model_info = litellm.get_model_info(model=litellm_model) @@ -16097,7 +16104,7 @@ def _enrich_model_info_with_litellm_data( # 3rd pass on the model, try seeing if we can find model but without the "/" in model cost map if litellm_model_info == {}: # use litellm_param model_name to get model_info - litellm_params = model.get("litellm_params", {}) + litellm_params = priced_model.get("litellm_params", {}) litellm_model = litellm_params.get("model", None) if litellm_model: split_model: Final = litellm_model.split("/") diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index b892dc97e5c..9d81f7b8b12 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -4431,6 +4431,28 @@ class TestModelInfoCostMapEchoFilter: "cost-map metadata must not be persisted from an unchanged /model/info echo" ) + def test_echo_of_an_expanded_wildcard_row_is_not_persisted_on_the_wildcard(self): + import litellm + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("openai/gpt-5.6") + echo = {**entry, "id": "dep-echo-wildcard", "db_model": True, "access_groups": ["prod"]} + db_model = Deployment( + model_name="my-openai/*", + litellm_params=LiteLLM_Params(model="openai/*"), + model_info=ModelInfo(id="dep-echo-wildcard"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_name="my-openai/gpt-5.6", model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert info["access_groups"] == ["prod"] + assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto the whole wildcard" + def test_an_edited_value_survives_the_echo_filter(self): import litellm from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 9993361d930..20721ca24cf 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -578,6 +578,203 @@ async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monke assert [model["model_name"] for model in result] == ["openai/gpt-4o"] +def test_model_info_prices_expanded_wildcard_rows_as_the_model_their_calls_route_to( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """A catch-all over Fireworks lists ``acme-chat``, which also names a DeepSeek entry, and a + prefixed OpenAI wildcard lists ``my-openai/gpt-acme``, which names no entry at all. Each row must + show the provider and price its calls are billed at.""" + from litellm.proxy.auth import model_checks + + catalog: Final = { + "fireworks_ai/accounts/fireworks/models/acme-chat": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "acme-chat": { + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9e-06, + "litellm_provider": "deepseek", + "mode": "chat", + }, + "gpt-acme": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "openai", + "mode": "chat", + }, + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) + _invalidate_model_cost_lowercase_map() + monkeypatch.setattr( + model_checks, + "get_provider_models", + lambda provider, litellm_params=None: { + "fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-chat"], + "openai": ["gpt-acme"], + }.get(provider, []), + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "*", + "litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/*", "api_key": "x"}, + "model_info": {"id": "fireworks-wildcard"}, + }, + { + "model_name": "my-openai/*", + "litellm_params": {"model": "openai/*", "api_key": "x"}, + "model_info": {"id": "openai-wildcard"}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + + with auth_as(): + response: Final = client.get("/model/info") + + assert response.status_code == 200, response.text + listed: Final = { + row["model_name"]: ( + row["litellm_params"]["model"], + row["model_info"].get("litellm_provider"), + row["model_info"].get("input_cost_per_token"), + row["model_info"].get("output_cost_per_token"), + ) + for row in response.json()["data"] + } + assert listed == { + "acme-chat": ("acme-chat", "fireworks_ai", 1e-07, 2e-07), + "my-openai/gpt-acme": ("my-openai/gpt-acme", "openai", 3e-07, 4e-07), + } + calls: Final = { + model_name: router.completion( + model=model_name, messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + for model_name in listed + } + assert {model_name: call._hidden_params["response_cost"] for model_name, call in calls.items()} == pytest.approx( + { + model_name: listed[model_name][2] * call.usage.prompt_tokens + + listed[model_name][3] * call.usage.completion_tokens + for model_name, call in calls.items() + } + ) + _invalidate_model_cost_lowercase_map() + + +def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """``acme-lite`` has a deployment of its own and ``my-openai/gpt-acme-*`` is a narrower wildcard, + so calls to those names never reach the catch-all or ``my-openai/*``, which still list them. Such a + row may go unpriced, but any price a row shows must be the one its calls are billed at.""" + from litellm.proxy.auth import model_checks + + catalog: Final = { + "fireworks_ai/accounts/fireworks/models/acme-lite": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "deepseek/acme-lite": { + "input_cost_per_token": 7e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "deepseek", + "mode": "chat", + }, + "gpt-acme-lite": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "openai", + "mode": "chat", + }, + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) + _invalidate_model_cost_lowercase_map() + monkeypatch.setattr( + model_checks, + "get_provider_models", + lambda provider, litellm_params=None: { + "fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-lite"], + "openai": ["gpt-acme-lite"], + }.get(provider, []), + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "*", + "litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/*", "api_key": "x"}, + "model_info": {"id": "fireworks-wildcard"}, + }, + { + "model_name": "acme-lite", + "litellm_params": {"model": "deepseek/acme-lite", "api_key": "x"}, + "model_info": {"id": "deepseek-acme-lite"}, + }, + { + "model_name": "my-openai/*", + "litellm_params": {"model": "openai/*", "api_key": "x"}, + "model_info": {"id": "openai-wildcard"}, + }, + { + "model_name": "my-openai/gpt-acme-*", + "litellm_params": {"model": "deepseek/acme-*", "api_key": "x"}, + "model_info": {"id": "deepseek-wildcard"}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + + with auth_as(): + response: Final = client.get("/model/info") + + assert response.status_code == 200, response.text + listed: Final = { + (row["model_info"]["id"], row["model_name"]): ( + row["model_info"].get("input_cost_per_token"), + row["model_info"].get("output_cost_per_token"), + ) + for row in response.json()["data"] + if "*" not in row["model_name"] + } + assert sorted(listed) == [ + ("deepseek-acme-lite", "acme-lite"), + ("fireworks-wildcard", "acme-lite"), + ("openai-wildcard", "my-openai/gpt-acme-lite"), + ] + assert listed[("deepseek-acme-lite", "acme-lite")] == (7e-07, 8e-07) + calls: Final = { + model_name: router.completion( + model=model_name, messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + for _, model_name in listed + } + priced_rows: Final = {key: prices for key, prices in listed.items() if prices[0] is not None} + assert { + key: calls[key[1]]._hidden_params["response_cost"] for key in priced_rows + } == pytest.approx( + { + key: input_cost * calls[key[1]].usage.prompt_tokens + output_cost * calls[key[1]].usage.completion_tokens + for key, (input_cost, output_cost) in priced_rows.items() + } + ) + _invalidate_model_cost_lowercase_map() + + # --------------------------------------------------------------------------- # GET /model/info — team BYOK scoping (issue #30983) # --------------------------------------------------------------------------- From cd7cb3b03ca858746f595cedb5f8b8eea3ddd3b2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:03:49 -0700 Subject: [PATCH 02/12] fix(proxy): read wildcard routing through the router registry entry points --- litellm/proxy/auth/model_checks.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 6042876d193..df8c46fb66c 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -17,6 +17,7 @@ from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.router import Router +from litellm.router_api import RouterRegistry from litellm.router_utils.fallback_event_handlers import get_fallback_model_group from litellm.router_utils.pattern_match_deployments import PatternMatchRouter from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params @@ -552,12 +553,10 @@ def _litellm_model(deployment: Mapping[str, object]) -> object: return litellm_params.get("model") if is_str_keyed_mapping(litellm_params) else None -def _router_deployment(llm_router: Router | None, model_id: object) -> Mapping[str, object] | None: - if llm_router is None or not isinstance(model_id, str): +def _router_deployment(registry: RouterRegistry, model_id: object) -> Mapping[str, object] | None: + if not isinstance(model_id, str): return None - deployment: Final = _ROUTER_DEPLOYMENT.validate_python( - llm_router.get_model_info(id=model_id) # pyright: ignore[reportUnknownMemberType] # narrowed below - ) + deployment: Final = _ROUTER_DEPLOYMENT.validate_python(registry.deployment_dict(model_id)) return deployment if is_str_keyed_mapping(deployment) else None @@ -583,17 +582,20 @@ def _deployment_routed_model(source: Mapping[str, object], listed_name: str) -> def wildcard_routed_model(row: Mapping[str, object], llm_router: Router | None) -> str | None: - source: Final = _router_deployment(llm_router, _model_info_field(row, "id")) - listed_name: Final = row.get("model_name") - if llm_router is None or source is None or not isinstance(listed_name, str): + if llm_router is None: return None - if _litellm_model(row) == _litellm_model(source) or listed_name in llm_router.model_names: + registry: Final = llm_router.registry() + source: Final = _router_deployment(registry, _model_info_field(row, "id")) + listed_name: Final = row.get("model_name") + if source is None or not isinstance(listed_name, str): + return None + if _litellm_model(row) == _litellm_model(source) or listed_name in registry.names(kind="group_set"): return None routed_model: Final = _deployment_routed_model(source, listed_name) return ( routed_model if routed_model is not None - and routed_model in llm_router.wildcard_routed_models(listed_name, _deployment_team_id(row)) + and routed_model in registry.names(listed_name, kind="wildcard", team_id=_deployment_team_id(row)) else None ) From c47a74293484bacb9c9561171664ae3bb6f613e8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:05:58 -0700 Subject: [PATCH 03/12] fix(proxy): price a wildcard row only when the router sends its name there Use the router's own route candidates for the listed name, so an alias, a routing group, a more specific pattern, or a team's own deployment keeps a wildcard row unpriced --- litellm/proxy/auth/model_checks.py | 29 ++--- .../proxy_server/test_routes_model_info.py | 104 +++++++++++++++++- 2 files changed, 108 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index df8c46fb66c..ad651b45f48 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -8,8 +8,6 @@ from collections.abc import Callable, Mapping, Sequence from types import MappingProxyType from typing import Final -from pydantic import TypeAdapter - import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor @@ -31,8 +29,6 @@ _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) -_ROUTER_DEPLOYMENT: Final = TypeAdapter(object) - def _check_wildcard_routing(model: str) -> bool: """ @@ -553,10 +549,8 @@ def _litellm_model(deployment: Mapping[str, object]) -> object: return litellm_params.get("model") if is_str_keyed_mapping(litellm_params) else None -def _router_deployment(registry: RouterRegistry, model_id: object) -> Mapping[str, object] | None: - if not isinstance(model_id, str): - return None - deployment: Final = _ROUTER_DEPLOYMENT.validate_python(registry.deployment_dict(model_id)) +def _router_deployment(registry: RouterRegistry, model_id: str) -> Mapping[str, object] | None: + deployment: Final[object] = registry.deployment_dict(model_id) return deployment if is_str_keyed_mapping(deployment) else None @@ -582,22 +576,17 @@ def _deployment_routed_model(source: Mapping[str, object], listed_name: str) -> def wildcard_routed_model(row: Mapping[str, object], llm_router: Router | None) -> str | None: - if llm_router is None: + model_id: Final = _model_info_field(row, "id") + listed_name: Final = row.get("model_name") + if llm_router is None or not isinstance(model_id, str) or not isinstance(listed_name, str): return None registry: Final = llm_router.registry() - source: Final = _router_deployment(registry, _model_info_field(row, "id")) - listed_name: Final = row.get("model_name") - if source is None or not isinstance(listed_name, str): + source: Final = _router_deployment(registry, model_id) + if source is None or _litellm_model(row) == _litellm_model(source): return None - if _litellm_model(row) == _litellm_model(source) or listed_name in registry.names(kind="group_set"): + if model_id not in registry.names(listed_name, kind="route_candidates", team_id=_deployment_team_id(row)): return None - routed_model: Final = _deployment_routed_model(source, listed_name) - return ( - routed_model - if routed_model is not None - and routed_model in registry.names(listed_name, kind="wildcard", team_id=_deployment_team_id(row)) - else None - ) + return _deployment_routed_model(source, listed_name) def _get_wildcard_models( diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 2738b0fde7e..0968ecb1d24 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -676,9 +676,10 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, ) -> None: - """``acme-lite`` has a deployment of its own and ``my-openai/gpt-acme-*`` is a narrower wildcard, - so calls to those names never reach the catch-all or ``my-openai/*``, which still list them. Such a - row may go unpriced, but any price a row shows must be the one its calls are billed at.""" + """``acme-lite`` has a deployment of its own, ``acme-pro`` is an alias of it, ``acme-max`` is a + routing group over it, and ``my-openai/gpt-acme-*`` is a narrower wildcard, so calls to those names + never reach the catch-all or ``my-openai/*``, which still list them. Such a row may go unpriced, but + any price a row shows must be the one its calls are billed at.""" from litellm.proxy.auth import model_checks catalog: Final = { @@ -688,6 +689,18 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( "litellm_provider": "fireworks_ai", "mode": "chat", }, + "fireworks_ai/accounts/fireworks/models/acme-pro": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 2.5e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "fireworks_ai/accounts/fireworks/models/acme-max": { + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 4.5e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, "deepseek/acme-lite": { "input_cost_per_token": 7e-07, "output_cost_per_token": 8e-07, @@ -707,7 +720,11 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( model_checks, "get_provider_models", lambda provider, litellm_params=None: { - "fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-lite"], + "fireworks_ai": [ + "fireworks_ai/accounts/fireworks/models/acme-lite", + "fireworks_ai/accounts/fireworks/models/acme-pro", + "fireworks_ai/accounts/fireworks/models/acme-max", + ], "openai": ["gpt-acme-lite"], }.get(provider, []), ) @@ -733,7 +750,9 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( "litellm_params": {"model": "deepseek/acme-*", "api_key": "x"}, "model_info": {"id": "deepseek-wildcard"}, }, - ] + ], + model_group_alias={"acme-pro": "acme-lite"}, + routing_groups=[{"group_name": "acme-max", "models": ["acme-lite"], "routing_strategy": "simple-shuffle"}], ) monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) @@ -753,7 +772,10 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( } assert sorted(listed) == [ ("deepseek-acme-lite", "acme-lite"), + ("deepseek-acme-lite", "acme-pro"), ("fireworks-wildcard", "acme-lite"), + ("fireworks-wildcard", "acme-max"), + ("fireworks-wildcard", "acme-pro"), ("openai-wildcard", "my-openai/gpt-acme-lite"), ] assert listed[("deepseek-acme-lite", "acme-lite")] == (7e-07, 8e-07) @@ -775,6 +797,78 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( _invalidate_model_cost_lowercase_map() +def test_model_info_prices_a_team_wildcard_row_by_the_route_its_team_takes( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """Team ``team-a`` owns ``team-fw/*`` over Fireworks and its own DeepSeek deployment published as + ``team-fw/acme-chat``, so the team's calls to that name skip the wildcard while ``team-fw/acme-lite`` + goes through it.""" + from litellm.proxy.auth import model_checks + + catalog: Final = { + "fireworks_ai/accounts/fireworks/models/acme-chat": { + "input_cost_per_token": 1e-07, + "output_cost_per_token": 2e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "fireworks_ai/accounts/fireworks/models/acme-lite": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) + _invalidate_model_cost_lowercase_map() + monkeypatch.setattr( + model_checks, + "get_provider_models", + lambda provider, litellm_params=None: { + "fireworks_ai": [ + "fireworks_ai/accounts/fireworks/models/acme-chat", + "fireworks_ai/accounts/fireworks/models/acme-lite", + ], + }.get(provider, []), + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "team-fw/*", + "litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/*", "api_key": "x"}, + "model_info": {"id": "team-wildcard", "team_id": "team-a"}, + }, + { + "model_name": "model_name_team-a_acme-chat", + "litellm_params": {"model": "deepseek/acme-chat", "api_key": "x"}, + "model_info": { + "id": "team-deepseek", + "team_id": "team-a", + "team_public_model_name": "team-fw/acme-chat", + }, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + + with auth_as(): + response: Final = client.get("/model/info") + + assert response.status_code == 200, response.text + wildcard_rows: Final = { + row["model_name"]: row["model_info"].get("input_cost_per_token") + for row in response.json()["data"] + if row["model_info"]["id"] == "team-wildcard" and "*" not in row["model_name"] + } + assert wildcard_rows == {"team-fw/acme-chat": None, "team-fw/acme-lite": 5e-07} + _invalidate_model_cost_lowercase_map() + + # --------------------------------------------------------------------------- # GET /model/info — team BYOK scoping (issue #30983) # --------------------------------------------------------------------------- From 8dbb033a36079daa29e9fed7d5842dd0b8a155a4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:24:44 -0700 Subject: [PATCH 04/12] fix(proxy): show no price on a wildcard row whose calls go to another deployment --- litellm/proxy/auth/model_checks.py | 22 +++++++----------- litellm/proxy/proxy_server.py | 8 +++---- .../proxy_server/test_routes_model_info.py | 23 +++++++++++++++++-- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index ad651b45f48..9c3e0f4b67c 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -565,28 +565,22 @@ def wildcard_filled_model(model_name_pattern: str, litellm_model: str, listed_na ) -def _deployment_routed_model(source: Mapping[str, object], listed_name: str) -> str | None: - source_pattern: Final = source.get("model_name") - source_model: Final = _litellm_model(source) - return ( - wildcard_filled_model(source_pattern, source_model, listed_name) - if isinstance(source_pattern, str) and isinstance(source_model, str) - else None - ) - - -def wildcard_routed_model(row: Mapping[str, object], llm_router: Router | None) -> str | None: +def wildcard_row_priced_model(row: Mapping[str, object], llm_router: Router | None) -> str | None: model_id: Final = _model_info_field(row, "id") listed_name: Final = row.get("model_name") if llm_router is None or not isinstance(model_id, str) or not isinstance(listed_name, str): return None registry: Final = llm_router.registry() source: Final = _router_deployment(registry, model_id) - if source is None or _litellm_model(row) == _litellm_model(source): + if source is None: + return None + source_pattern: Final = source.get("model_name") + source_model: Final = _litellm_model(source) + if not isinstance(source_pattern, str) or not isinstance(source_model, str) or _litellm_model(row) == source_model: return None if model_id not in registry.names(listed_name, kind="route_candidates", team_id=_deployment_team_id(row)): - return None - return _deployment_routed_model(source, listed_name) + return source_model + return wildcard_filled_model(source_pattern, source_model, listed_name) or source_model def _get_wildcard_models( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index daf2f226e46..d73dd268b93 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -443,7 +443,7 @@ from litellm.proxy.auth.model_checks import ( get_key_models, get_mcp_server_ids, get_team_models, - wildcard_routed_model, + wildcard_row_priced_model, ) from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # legacy module exports @@ -16145,11 +16145,9 @@ def _enrich_model_info_with_litellm_data( # read litellm model_prices_and_context_window.json to get the following: # input_cost_per_token, output_cost_per_token, max_tokens - routed_model: Final = wildcard_routed_model(model, llm_router) + priced_as: Final = wildcard_row_priced_model(model, llm_router) priced_model: Final = ( - model - if routed_model is None - else {**model, "litellm_params": {**model["litellm_params"], "model": routed_model}} + model if priced_as is None else {**model, "litellm_params": {**model["litellm_params"], "model": priced_as}} ) litellm_model_info = get_litellm_model_info(model=priced_model) diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 0968ecb1d24..08947a54a0c 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -678,8 +678,9 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( ) -> None: """``acme-lite`` has a deployment of its own, ``acme-pro`` is an alias of it, ``acme-max`` is a routing group over it, and ``my-openai/gpt-acme-*`` is a narrower wildcard, so calls to those names - never reach the catch-all or ``my-openai/*``, which still list them. Such a row may go unpriced, but - any price a row shows must be the one its calls are billed at.""" + never reach the catch-all or ``my-openai/*``, which still list them. Such a row shows no price, even + where another provider's model goes by the same name, and every price shown is the one calls are + billed at.""" from litellm.proxy.auth import model_checks catalog: Final = { @@ -713,6 +714,18 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( "litellm_provider": "openai", "mode": "chat", }, + "acme-pro": { + "input_cost_per_token": 9e-07, + "output_cost_per_token": 1e-06, + "litellm_provider": "deepseek", + "mode": "chat", + }, + "acme-max": { + "input_cost_per_token": 1.1e-06, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "deepseek", + "mode": "chat", + }, } monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) _invalidate_model_cost_lowercase_map() @@ -779,6 +792,12 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( ("openai-wildcard", "my-openai/gpt-acme-lite"), ] assert listed[("deepseek-acme-lite", "acme-lite")] == (7e-07, 8e-07) + assert {key: prices for key, prices in listed.items() if key[0] != "deepseek-acme-lite"} == { + ("fireworks-wildcard", "acme-lite"): (None, None), + ("fireworks-wildcard", "acme-max"): (None, None), + ("fireworks-wildcard", "acme-pro"): (None, None), + ("openai-wildcard", "my-openai/gpt-acme-lite"): (None, None), + } calls: Final = { model_name: router.completion( model=model_name, messages=[{"role": "user", "content": "hi"}], mock_response="hello there" From fa4b26fc9fd9357730e7b04b1e3f9b7ce094c464 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:03:32 -0700 Subject: [PATCH 05/12] fix(proxy): keep the listed price on catch-all wildcard rows and match the edit path to the read path --- litellm/proxy/auth/model_checks.py | 35 ++++- .../model_management_endpoints.py | 45 ++++-- litellm/proxy/proxy_server.py | 31 +--- .../proxy_server/test_routes_model_info.py | 146 ++++++++++++++++++ 4 files changed, 216 insertions(+), 41 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 9c3e0f4b67c..afe9b3e7229 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -13,6 +13,7 @@ from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth +from litellm.proxy.common_utils.token_count_settings import cost_map_entry from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.router import Router from litellm.router_api import RouterRegistry @@ -578,9 +579,37 @@ def wildcard_row_priced_model(row: Mapping[str, object], llm_router: Router | No source_model: Final = _litellm_model(source) if not isinstance(source_pattern, str) or not isinstance(source_model, str) or _litellm_model(row) == source_model: return None - if model_id not in registry.names(listed_name, kind="route_candidates", team_id=_deployment_team_id(row)): - return source_model - return wildcard_filled_model(source_pattern, source_model, listed_name) or source_model + return wildcard_listed_priced_model( + source_pattern, + source_model, + listed_name, + routes_through_wildcard=routes_through_deployment(llm_router, model_id, listed_name, _deployment_team_id(row)), + ) + + +def wildcard_listed_priced_model( + model_name_pattern: str, wildcard_model: str, listed_name: str, routes_through_wildcard: bool +) -> str | None: + if not routes_through_wildcard: + return wildcard_model + filled: Final = wildcard_filled_model(model_name_pattern, wildcard_model, listed_name) + if filled is None: + return wildcard_model + return None if _repeats_listed_provider_prefix(filled, listed_name) else filled + + +def _repeats_listed_provider_prefix(filled_model: str, listed_name: str) -> bool: + listed_provider, separator, _ = listed_name.partition("/") + return separator != "" and filled_model == f"{listed_provider}/{listed_name}" + + +def routes_through_deployment(llm_router: Router, model_id: str, listed_name: str, team_id: str | None) -> bool: + return model_id in llm_router.registry().names(listed_name, kind="route_candidates", team_id=team_id) + + +def model_info_cost_map_entry(model: str) -> Mapping[str, object]: + provider, _, _ = model.partition("/") + return cost_map_entry(model) or cost_map_entry(model.rsplit("/", 1)[-1], custom_llm_provider=provider) def _get_wildcard_models( diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index ac4123038ee..152c14ef038 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -15,7 +15,7 @@ import datetime import itertools import json from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence -from contextlib import AbstractAsyncContextManager, asynccontextmanager, suppress +from contextlib import AbstractAsyncContextManager, asynccontextmanager from dataclasses import dataclass from fnmatch import fnmatchcase from functools import partial @@ -61,7 +61,11 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY -from litellm.proxy.auth.model_checks import wildcard_filled_model +from litellm.proxy.auth.model_checks import ( + model_info_cost_map_entry, + routes_through_deployment, + wildcard_listed_priced_model, +) from litellm.proxy.auth.team_grants import team_model_aliases from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.config_sync_pubsub import ( @@ -1266,23 +1270,33 @@ def _ptu_priced_deployment(model_params: Deployment) -> Deployment: ) -def _live_cost_map_model(db_model: Deployment, listed_name: str | None) -> str | None: +def _live_cost_map_model(db_model: Deployment, listed_name: str | None, llm_router: Router | None) -> str | None: stored_model: Final = _decrypted_model(db_model.litellm_params.model) if stored_model is None or listed_name is None: return stored_model - return wildcard_filled_model(db_model.model_name, stored_model, listed_name) or stored_model + model_id: Final = db_model.model_info.id + priced_as: Final = wildcard_listed_priced_model( + db_model.model_name, + stored_model, + listed_name, + routes_through_wildcard=llm_router is None + or model_id is None + or routes_through_deployment(llm_router, model_id, listed_name, db_model.model_info.team_id), + ) + return listed_name if priced_as is None else priced_as def _cost_map_entry( - db_model: Deployment, incoming_model_info: Mapping[str, object], listed_name: str | None + db_model: Deployment, + incoming_model_info: Mapping[str, object], + listed_name: str | None, + llm_router: Router | None, ) -> Mapping[str, object]: base_model: Final = incoming_model_info.get("base_model") - lookup: Final = base_model if isinstance(base_model, str) else _live_cost_map_model(db_model, listed_name) - if lookup is None: - return MappingProxyType({}) - with suppress(Exception): - return MappingProxyType(dict(litellm.get_model_info(model=lookup))) - return MappingProxyType({}) + lookup: Final = ( + base_model if isinstance(base_model, str) else _live_cost_map_model(db_model, listed_name, llm_router) + ) + return MappingProxyType({}) if lookup is None else model_info_cost_map_entry(lookup) LoadedCatalog: TypeAlias = Callable[[], Mapping[str, Mapping[str, object]]] @@ -1305,6 +1319,7 @@ def update_db_model( db_model: Deployment, updated_patch: updateDeployment, loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map, + llm_router: Router | None = None, ) -> PrismaCompatibleUpdateDBModel: if updated_patch.model_info is not None: _raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True)) @@ -1336,7 +1351,7 @@ def update_db_model( incoming_model_info: Final = updated_patch.model_info.model_dump(exclude_none=True) echoed_fields: Final = echoed_cost_map_fields( incoming_model_info, - _cost_map_entry(db_model, incoming_model_info, updated_patch.model_name), + _cost_map_entry(db_model, incoming_model_info, updated_patch.model_name, llm_router), _loaded_catalog_entry(incoming_model_info, loaded_catalog), ) stored_model_info_fields: Final[Mapping[str, object]] = without_server_derived_pricing(incoming_model_info) @@ -1689,6 +1704,7 @@ async def patch_model( user_api_key_dict=user_api_key_dict, prisma_client=prisma_client, write_row=write_row, + llm_router=llm_router, ) if updated_model is None: @@ -2044,6 +2060,7 @@ async def _update_team_model_in_db( user_api_key_dict: UserAPIKeyAuth, prisma_client: PrismaClient, write_row: Callable[[PrismaCompatibleUpdateDBModel], Awaitable[_RowT]], + llm_router: Router | None = None, ) -> _RowT: """ Handle team model updates with proper alias management. @@ -2088,7 +2105,7 @@ async def _update_team_model_in_db( # No team_id in patch, proceed with standard update if patch_team_id is None: - return await write_row(update_db_model(db_model=db_model, updated_patch=patch_data)) + return await write_row(update_db_model(db_model=db_model, updated_patch=patch_data, llm_router=llm_router)) # Determine public model name public_model_name: Final = _get_public_model_name( @@ -2109,7 +2126,7 @@ async def _update_team_model_in_db( ), } ) - row: Final = await write_row(update_db_model(db_model=db_model, updated_patch=team_patch)) + row: Final = await write_row(update_db_model(db_model=db_model, updated_patch=team_patch, llm_router=llm_router)) if is_new_team_assignment: await _setup_new_team_model_assignment( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d73dd268b93..5f545a558b9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -443,6 +443,7 @@ from litellm.proxy.auth.model_checks import ( get_key_models, get_mcp_server_ids, get_team_models, + model_info_cost_map_entry, wildcard_row_priced_model, ) from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy @@ -16149,30 +16150,12 @@ def _enrich_model_info_with_litellm_data( priced_model: Final = ( model if priced_as is None else {**model, "litellm_params": {**model["litellm_params"], "model": priced_as}} ) - litellm_model_info = get_litellm_model_info(model=priced_model) - - # 2nd pass on the model, try seeing if we can find model in litellm model_cost map - if litellm_model_info == {}: - # use litellm_param model_name to get model_info - litellm_params = priced_model.get("litellm_params", {}) - litellm_model = litellm_params.get("model", None) - try: - litellm_model_info = litellm.get_model_info(model=litellm_model) - except Exception: - litellm_model_info = {} - # 3rd pass on the model, try seeing if we can find model but without the "/" in model cost map - if litellm_model_info == {}: - # use litellm_param model_name to get model_info - litellm_params = priced_model.get("litellm_params", {}) - litellm_model = litellm_params.get("model", None) - if litellm_model: - split_model: Final = litellm_model.split("/") - if len(split_model) > 0: - litellm_model = split_model[-1] - try: - litellm_model_info = litellm.get_model_info(model=litellm_model, custom_llm_provider=split_model[0]) - except Exception: - litellm_model_info = {} + priced_litellm_model: Final = priced_model.get("litellm_params", {}).get("model") + litellm_model_info: Final = get_litellm_model_info(model=priced_model) or ( + model_info_cost_map_entry(priced_litellm_model) + if isinstance(priced_litellm_model, str) + else MappingProxyType({}) + ) discovered_model_info: Final = ( llm_router.registry().discovered_model_info(model_info.get("id")) if llm_router is not None diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 08947a54a0c..35b7df7eda8 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -888,6 +888,152 @@ def test_model_info_prices_a_team_wildcard_row_by_the_route_its_team_takes( _invalidate_model_cost_lowercase_map() +def _catch_all_and_shadowed_wildcard_router(monkeypatch: pytest.MonkeyPatch) -> litellm.Router: + from litellm.proxy.auth import model_checks + + catalog: Final = { + "gpt-5.6-acme": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "openai", + "mode": "chat", + }, + "fireworks_ai/accounts/fireworks/models/acme-pro": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 2.5e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "deepseek/acme-lite": { + "input_cost_per_token": 7e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "deepseek", + "mode": "chat", + }, + } + monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) + _invalidate_model_cost_lowercase_map() + monkeypatch.setattr( + model_checks, + "get_provider_models", + lambda provider, litellm_params=None: { + "openai": ["gpt-5.6-acme"], + "fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-pro"], + }.get(provider, []), + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "*", + "litellm_params": {"model": "openai/*", "api_key": "x"}, + "model_info": {"id": "openai-catch-all"}, + }, + { + "model_name": "my-fw/*", + "litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/*", "api_key": "x"}, + "model_info": {"id": "fireworks-wildcard"}, + }, + { + "model_name": "acme-lite", + "litellm_params": {"model": "deepseek/acme-lite", "api_key": "x"}, + "model_info": {"id": "deepseek-acme-lite"}, + }, + ], + model_group_alias={"my-fw/acme-pro": "acme-lite"}, + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + return router + + +def _expanded_wildcard_rows(client: TestClient) -> dict[str, dict[str, object]]: + response: Final = client.get("/model/info") + assert response.status_code == 200, response.text + return { + row["model_name"]: row + for row in response.json()["data"] + if row["model_info"]["id"] != "deepseek-acme-lite" and "*" not in row["model_name"] + } + + +def test_model_info_prices_a_catch_all_row_named_with_its_provider_as_the_listed_model( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """A catch-all over ``openai/*`` lists ``openai/gpt-5.6-acme``. Filling the wildcard with that name repeats + the provider, so the row must keep the price of the model it names, the one calls to ``gpt-5.6-acme`` are + billed at.""" + router: Final = _catch_all_and_shadowed_wildcard_router(monkeypatch) + + with auth_as(): + rows: Final = _expanded_wildcard_rows(client) + + shown: Final = rows["openai/gpt-5.6-acme"]["model_info"] + assert (shown.get("input_cost_per_token"), shown.get("output_cost_per_token")) == (3e-07, 4e-07) + call: Final = router.completion( + model="gpt-5.6-acme", messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + assert call._hidden_params["response_cost"] == pytest.approx( + shown["input_cost_per_token"] * call.usage.prompt_tokens + + shown["output_cost_per_token"] * call.usage.completion_tokens + ) + _invalidate_model_cost_lowercase_map() + + +def test_saving_a_replayed_model_info_wildcard_row_stores_none_of_its_shown_metadata( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """An API client that saves a ``/model/info`` row back unchanged, apart from one edit, must not pin the + cost-map metadata the row showed onto the wildcard, whether the row is a catch-all name carrying its + provider or a name an alias sends to another deployment.""" + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment + + router: Final = _catch_all_and_shadowed_wildcard_router(monkeypatch) + with auth_as(): + rows: Final = _expanded_wildcard_rows(client) + + assert sorted(rows) == ["my-fw/acme-pro", "openai/gpt-5.6-acme"] + sources: Final = { + deployment["model_info"]["id"]: deployment for deployment in router.get_model_list() or [] + } + stored: Final = { + name: json.loads( + update_db_model( + db_model=Deployment( + model_name=sources[row["model_info"]["id"]]["model_name"], + litellm_params=LiteLLM_Params(model=sources[row["model_info"]["id"]]["litellm_params"]["model"]), + model_info=ModelInfo(id=row["model_info"]["id"]), + ), + updated_patch=updateDeployment( + model_name=name, model_info=ModelInfo(**{**row["model_info"], "access_groups": ["prod"]}) + ), + llm_router=router, + )["model_info"] + ) + for name, row in rows.items() + } + shown_metadata: Final = { + name: set(row["model_info"]) - set(sources[row["model_info"]["id"]]["model_info"]) for name, row in rows.items() + } + assert all(shown_metadata.values()), shown_metadata + assert {name: info["access_groups"] for name, info in stored.items()} == { + "my-fw/acme-pro": ["prod"], + "openai/gpt-5.6-acme": ["prod"], + } + assert {name: shown_metadata[name] & set(info) for name, info in stored.items()} == { + "my-fw/acme-pro": set(), + "openai/gpt-5.6-acme": set(), + } + _invalidate_model_cost_lowercase_map() + + # --------------------------------------------------------------------------- # GET /model/info — team BYOK scoping (issue #30983) # --------------------------------------------------------------------------- From 2083957b33a2f26a0ca0ed6fbef3f8a6a433c564 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:08:15 -0700 Subject: [PATCH 06/12] fix(proxy): price a wildcard row aliased to a sibling name as the alias target --- litellm/proxy/auth/model_checks.py | 20 ++++--- .../model_management_endpoints.py | 6 +- .../proxy_server/test_routes_model_info.py | 58 +++++++++++++++---- 3 files changed, 61 insertions(+), 23 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index afe9b3e7229..e5f9e5c7c1a 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -582,25 +582,29 @@ def wildcard_row_priced_model(row: Mapping[str, object], llm_router: Router | No return wildcard_listed_priced_model( source_pattern, source_model, - listed_name, + routed_model_name(llm_router, listed_name), routes_through_wildcard=routes_through_deployment(llm_router, model_id, listed_name, _deployment_team_id(row)), ) def wildcard_listed_priced_model( - model_name_pattern: str, wildcard_model: str, listed_name: str, routes_through_wildcard: bool -) -> str | None: + model_name_pattern: str, wildcard_model: str, routed_name: str, routes_through_wildcard: bool +) -> str: if not routes_through_wildcard: return wildcard_model - filled: Final = wildcard_filled_model(model_name_pattern, wildcard_model, listed_name) + filled: Final = wildcard_filled_model(model_name_pattern, wildcard_model, routed_name) if filled is None: return wildcard_model - return None if _repeats_listed_provider_prefix(filled, listed_name) else filled + return routed_name if _repeats_provider_prefix(filled, routed_name) else filled -def _repeats_listed_provider_prefix(filled_model: str, listed_name: str) -> bool: - listed_provider, separator, _ = listed_name.partition("/") - return separator != "" and filled_model == f"{listed_provider}/{listed_name}" +def _repeats_provider_prefix(filled_model: str, routed_name: str) -> bool: + provider, separator, _ = routed_name.partition("/") + return separator != "" and filled_model == f"{provider}/{routed_name}" + + +def routed_model_name(llm_router: Router, listed_name: str) -> str: + return llm_router.registry().resolve(listed_name, kind="alias") or listed_name def routes_through_deployment(llm_router: Router, model_id: str, listed_name: str, team_id: str | None) -> bool: diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 152c14ef038..143f3869d19 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -63,6 +63,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY from litellm.proxy.auth.model_checks import ( model_info_cost_map_entry, + routed_model_name, routes_through_deployment, wildcard_listed_priced_model, ) @@ -1275,15 +1276,14 @@ def _live_cost_map_model(db_model: Deployment, listed_name: str | None, llm_rout if stored_model is None or listed_name is None: return stored_model model_id: Final = db_model.model_info.id - priced_as: Final = wildcard_listed_priced_model( + return wildcard_listed_priced_model( db_model.model_name, stored_model, - listed_name, + listed_name if llm_router is None else routed_model_name(llm_router, listed_name), routes_through_wildcard=llm_router is None or model_id is None or routes_through_deployment(llm_router, model_id, listed_name, db_model.model_info.team_id), ) - return listed_name if priced_as is None else priced_as def _cost_map_entry( diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 35b7df7eda8..5c900a6eb5b 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -904,6 +904,18 @@ def _catch_all_and_shadowed_wildcard_router(monkeypatch: pytest.MonkeyPatch) -> "litellm_provider": "fireworks_ai", "mode": "chat", }, + "fireworks_ai/accounts/fireworks/models/acme-max": { + "input_cost_per_token": 9e-07, + "output_cost_per_token": 9.5e-07, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, + "fireworks_ai/accounts/fireworks/models/acme-mini": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 6e-08, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, "deepseek/acme-lite": { "input_cost_per_token": 7e-07, "output_cost_per_token": 8e-07, @@ -918,7 +930,11 @@ def _catch_all_and_shadowed_wildcard_router(monkeypatch: pytest.MonkeyPatch) -> "get_provider_models", lambda provider, litellm_params=None: { "openai": ["gpt-5.6-acme"], - "fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-pro"], + "fireworks_ai": [ + "fireworks_ai/accounts/fireworks/models/acme-pro", + "fireworks_ai/accounts/fireworks/models/acme-max", + "fireworks_ai/accounts/fireworks/models/acme-mini", + ], }.get(provider, []), ) router: Final = litellm.Router( @@ -939,7 +955,7 @@ def _catch_all_and_shadowed_wildcard_router(monkeypatch: pytest.MonkeyPatch) -> "model_info": {"id": "deepseek-acme-lite"}, }, ], - model_group_alias={"my-fw/acme-pro": "acme-lite"}, + model_group_alias={"my-fw/acme-pro": "acme-lite", "my-fw/acme-max": "my-fw/acme-mini"}, ) monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) @@ -983,6 +999,30 @@ def test_model_info_prices_a_catch_all_row_named_with_its_provider_as_the_listed _invalidate_model_cost_lowercase_map() +def test_model_info_prices_a_wildcard_row_aliased_to_a_sibling_name_as_the_alias_target( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """An alias from ``my-fw/acme-max`` to ``my-fw/acme-mini`` still reaches the same wildcard, but calls run + ``acme-mini``, so the ``my-fw/acme-max`` row must show the price those calls are billed at.""" + router: Final = _catch_all_and_shadowed_wildcard_router(monkeypatch) + + with auth_as(): + rows: Final = _expanded_wildcard_rows(client) + + shown: Final = rows["my-fw/acme-max"]["model_info"] + call: Final = router.completion( + model="my-fw/acme-max", messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + assert call._hidden_params["response_cost"] == pytest.approx( + shown["input_cost_per_token"] * call.usage.prompt_tokens + + shown["output_cost_per_token"] * call.usage.completion_tokens + ) + _invalidate_model_cost_lowercase_map() + + def test_saving_a_replayed_model_info_wildcard_row_stores_none_of_its_shown_metadata( client: TestClient, auth_as: Callable[[], AbstractContextManager[object]], @@ -991,7 +1031,7 @@ def test_saving_a_replayed_model_info_wildcard_row_stores_none_of_its_shown_meta ) -> None: """An API client that saves a ``/model/info`` row back unchanged, apart from one edit, must not pin the cost-map metadata the row showed onto the wildcard, whether the row is a catch-all name carrying its - provider or a name an alias sends to another deployment.""" + provider, a name an alias sends to another deployment, or a name an alias sends to a sibling name.""" from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment @@ -999,7 +1039,7 @@ def test_saving_a_replayed_model_info_wildcard_row_stores_none_of_its_shown_meta with auth_as(): rows: Final = _expanded_wildcard_rows(client) - assert sorted(rows) == ["my-fw/acme-pro", "openai/gpt-5.6-acme"] + assert sorted(rows) == ["my-fw/acme-max", "my-fw/acme-mini", "my-fw/acme-pro", "openai/gpt-5.6-acme"] sources: Final = { deployment["model_info"]["id"]: deployment for deployment in router.get_model_list() or [] } @@ -1023,14 +1063,8 @@ def test_saving_a_replayed_model_info_wildcard_row_stores_none_of_its_shown_meta name: set(row["model_info"]) - set(sources[row["model_info"]["id"]]["model_info"]) for name, row in rows.items() } assert all(shown_metadata.values()), shown_metadata - assert {name: info["access_groups"] for name, info in stored.items()} == { - "my-fw/acme-pro": ["prod"], - "openai/gpt-5.6-acme": ["prod"], - } - assert {name: shown_metadata[name] & set(info) for name, info in stored.items()} == { - "my-fw/acme-pro": set(), - "openai/gpt-5.6-acme": set(), - } + assert {name: info["access_groups"] for name, info in stored.items()} == dict.fromkeys(rows, ["prod"]) + assert {name: shown_metadata[name] & set(info) for name, info in stored.items()} == dict.fromkeys(rows, set()) _invalidate_model_cost_lowercase_map() From 8e329d4bab369ac3c816d974686f2b3292cf5e10 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:46:03 -0700 Subject: [PATCH 07/12] fix(proxy): price wildcard rows through the caller's alias renames and show nothing for rows another deployment takes --- litellm/proxy/auth/model_checks.py | 26 ++++-- .../proxy/common_utils/model_listing_utils.py | 5 ++ .../model_management_endpoints.py | 44 +++++++++-- litellm/proxy/proxy_server.py | 79 +++++++++++-------- .../test_model_management_endpoints.py | 41 ++++++++++ .../proxy_server/test_routes_model_info.py | 70 ++++++++++++++-- .../test_team_model_name_translation.py | 4 +- 7 files changed, 217 insertions(+), 52 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index e5f9e5c7c1a..331dfc4b19c 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -5,6 +5,7 @@ import functools import itertools import re from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass from types import MappingProxyType from typing import Final @@ -13,6 +14,7 @@ from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth +from litellm.proxy.common_utils.model_listing_utils import CallerAliases, served_model from litellm.proxy.common_utils.token_count_settings import cost_map_entry from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.router import Router @@ -566,7 +568,20 @@ def wildcard_filled_model(model_name_pattern: str, litellm_model: str, listed_na ) -def wildcard_row_priced_model(row: Mapping[str, object], llm_router: Router | None) -> str | None: +GLOBAL_ALIASES_ONLY: Final = CallerAliases(team=None, key=None) + + +@dataclass(frozen=True, slots=True) +class RoutedElsewhere: + pass + + +ROUTED_ELSEWHERE: Final = RoutedElsewhere() + + +def wildcard_row_priced_model( + row: Mapping[str, object], llm_router: Router | None, caller_aliases: CallerAliases = GLOBAL_ALIASES_ONLY +) -> str | RoutedElsewhere | None: model_id: Final = _model_info_field(row, "id") listed_name: Final = row.get("model_name") if llm_router is None or not isinstance(model_id, str) or not isinstance(listed_name, str): @@ -579,19 +594,20 @@ def wildcard_row_priced_model(row: Mapping[str, object], llm_router: Router | No source_model: Final = _litellm_model(source) if not isinstance(source_pattern, str) or not isinstance(source_model, str) or _litellm_model(row) == source_model: return None + served_name: Final = served_model(listed_name, caller_aliases) return wildcard_listed_priced_model( source_pattern, source_model, - routed_model_name(llm_router, listed_name), - routes_through_wildcard=routes_through_deployment(llm_router, model_id, listed_name, _deployment_team_id(row)), + routed_model_name(llm_router, served_name), + routes_through_wildcard=routes_through_deployment(llm_router, model_id, served_name, _deployment_team_id(row)), ) def wildcard_listed_priced_model( model_name_pattern: str, wildcard_model: str, routed_name: str, routes_through_wildcard: bool -) -> str: +) -> str | RoutedElsewhere: if not routes_through_wildcard: - return wildcard_model + return ROUTED_ELSEWHERE filled: Final = wildcard_filled_model(model_name_pattern, wildcard_model, routed_name) if filled is None: return wildcard_model diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py index ad4c3bd8770..2f7d22afdd1 100644 --- a/litellm/proxy/common_utils/model_listing_utils.py +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -287,6 +287,11 @@ def _dispatch(model_id: str, layers: Sequence[_AliasLayer]) -> tuple[str, bool]: return reduce(_through_layer, layers, (model_id, False)) +def served_model(model_id: str, aliases: CallerAliases) -> str: + target, _ = _dispatch(model_id, _alias_layers(aliases)) + return target + + def _lookup_id(name: str, lookup_by_response: Mapping[str, str], lookup_ids: Container[str]) -> str | None: if name in lookup_by_response: return lookup_by_response[name] diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 143f3869d19..774f1216084 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -62,6 +62,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY from litellm.proxy.auth.model_checks import ( + GLOBAL_ALIASES_ONLY, model_info_cost_map_entry, routed_model_name, routes_through_deployment, @@ -83,6 +84,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) +from litellm.proxy.common_utils.model_listing_utils import CallerAliases, served_model from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient @@ -1271,19 +1273,23 @@ def _ptu_priced_deployment(model_params: Deployment) -> Deployment: ) -def _live_cost_map_model(db_model: Deployment, listed_name: str | None, llm_router: Router | None) -> str | None: +def _live_cost_map_model( + db_model: Deployment, listed_name: str | None, llm_router: Router | None, caller_aliases: CallerAliases +) -> str | None: stored_model: Final = _decrypted_model(db_model.litellm_params.model) if stored_model is None or listed_name is None: return stored_model model_id: Final = db_model.model_info.id - return wildcard_listed_priced_model( + served_name: Final = served_model(listed_name, caller_aliases) + priced: Final = wildcard_listed_priced_model( db_model.model_name, stored_model, - listed_name if llm_router is None else routed_model_name(llm_router, listed_name), + served_name if llm_router is None else routed_model_name(llm_router, served_name), routes_through_wildcard=llm_router is None or model_id is None - or routes_through_deployment(llm_router, model_id, listed_name, db_model.model_info.team_id), + or routes_through_deployment(llm_router, model_id, served_name, db_model.model_info.team_id), ) + return priced if isinstance(priced, str) else stored_model def _cost_map_entry( @@ -1291,10 +1297,13 @@ def _cost_map_entry( incoming_model_info: Mapping[str, object], listed_name: str | None, llm_router: Router | None, + caller_aliases: CallerAliases, ) -> Mapping[str, object]: base_model: Final = incoming_model_info.get("base_model") lookup: Final = ( - base_model if isinstance(base_model, str) else _live_cost_map_model(db_model, listed_name, llm_router) + base_model + if isinstance(base_model, str) + else _live_cost_map_model(db_model, listed_name, llm_router, caller_aliases) ) return MappingProxyType({}) if lookup is None else model_info_cost_map_entry(lookup) @@ -1320,6 +1329,8 @@ def update_db_model( updated_patch: updateDeployment, loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map, llm_router: Router | None = None, + caller_aliases: CallerAliases = GLOBAL_ALIASES_ONLY, + listed_name: str | None = None, ) -> PrismaCompatibleUpdateDBModel: if updated_patch.model_info is not None: _raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True)) @@ -1351,7 +1362,9 @@ def update_db_model( incoming_model_info: Final = updated_patch.model_info.model_dump(exclude_none=True) echoed_fields: Final = echoed_cost_map_fields( incoming_model_info, - _cost_map_entry(db_model, incoming_model_info, updated_patch.model_name, llm_router), + _cost_map_entry( + db_model, incoming_model_info, listed_name or updated_patch.model_name, llm_router, caller_aliases + ), _loaded_catalog_entry(incoming_model_info, loaded_catalog), ) stored_model_info_fields: Final[Mapping[str, object]] = without_server_derived_pricing(incoming_model_info) @@ -1535,6 +1548,7 @@ async def patch_model( llm_router, premium_user, prisma_client, + request_caller_aliases, store_model_in_db, user_api_key_cache, ) @@ -1705,6 +1719,7 @@ async def patch_model( prisma_client=prisma_client, write_row=write_row, llm_router=llm_router, + caller_aliases=await request_caller_aliases(user_api_key_dict), ) if updated_model is None: @@ -2061,6 +2076,7 @@ async def _update_team_model_in_db( prisma_client: PrismaClient, write_row: Callable[[PrismaCompatibleUpdateDBModel], Awaitable[_RowT]], llm_router: Router | None = None, + caller_aliases: CallerAliases = GLOBAL_ALIASES_ONLY, ) -> _RowT: """ Handle team model updates with proper alias management. @@ -2105,7 +2121,11 @@ async def _update_team_model_in_db( # No team_id in patch, proceed with standard update if patch_team_id is None: - return await write_row(update_db_model(db_model=db_model, updated_patch=patch_data, llm_router=llm_router)) + return await write_row( + update_db_model( + db_model=db_model, updated_patch=patch_data, llm_router=llm_router, caller_aliases=caller_aliases + ) + ) # Determine public model name public_model_name: Final = _get_public_model_name( @@ -2126,7 +2146,15 @@ async def _update_team_model_in_db( ), } ) - row: Final = await write_row(update_db_model(db_model=db_model, updated_patch=team_patch, llm_router=llm_router)) + row: Final = await write_row( + update_db_model( + db_model=db_model, + updated_patch=team_patch, + llm_router=llm_router, + caller_aliases=caller_aliases, + listed_name=patch_data.model_name, + ) + ) if is_new_team_assignment: await _setup_new_team_model_assignment( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5f545a558b9..2227155fe08 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -437,6 +437,8 @@ from litellm.proxy.auth.master_key_boot_check import ( with_stored_secrets_counted, ) from litellm.proxy.auth.model_checks import ( + GLOBAL_ALIASES_ONLY, + RoutedElsewhere, expand_wildcard_deployments_for_model_info, get_all_fallbacks, get_complete_model_list, @@ -516,6 +518,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401, RUF10 from litellm.proxy.common_utils.load_config_utils import get_config_from_bucket from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations from litellm.proxy.common_utils.model_listing_utils import ( + CallerAliases, ClaudeCodeRoutingNames, DeployedModelGroups, TeamModelNameTranslator, @@ -12408,16 +12411,7 @@ async def model_list( http_request is not None and http_request.headers.get("anthropic-version") is not None ) client_headers: Final[Mapping[str, str]] = http_request.headers if http_request is not None else _EMPTY_HEADERS - caller_router_settings: Final = await proxy_config.get_hierarchical_router_settings( - user_api_key_dict, prisma_client, proxy_logging_obj - ) - caller_aliases: Final = caller_alias_maps( - user_api_key_dict.aliases, - user_api_key_dict.team_model_aliases, - user_api_key_dict.team_id, - team_id, - caller_router_settings.get("model_group_alias") if isinstance(caller_router_settings, Mapping) else None, - ) + caller_aliases: Final = await request_caller_aliases(user_api_key_dict, team_id) routing_names: Final = ClaudeCodeRoutingNames( llm_router, team_id or user_api_key_dict.team_id, @@ -12683,16 +12677,7 @@ async def model_info( ) hidden_names: Final = blocked_names | unhealthy_names internal_to_public: Final = TeamModelNameTranslator.build_internal_to_public_map(llm_router, settings) - caller_router_settings: Final = await proxy_config.get_hierarchical_router_settings( - user_api_key_dict, prisma_client, proxy_logging_obj - ) - caller_aliases: Final = caller_alias_maps( - user_api_key_dict.aliases, - user_api_key_dict.team_model_aliases, - user_api_key_dict.team_id, - team_id, - caller_router_settings.get("model_group_alias") if isinstance(caller_router_settings, Mapping) else None, - ) + caller_aliases: Final = await request_caller_aliases(user_api_key_dict, team_id) callback_hidden_names: Final = await _names_hidden_by_listing_callbacks( user_api_key_dict, tuple( @@ -16119,7 +16104,10 @@ async def get_all_team_and_direct_access_models( def _enrich_model_info_with_litellm_data( - model: dict[str, Any], debug: bool = False, llm_router: Router | None = None + model: dict[str, Any], + debug: bool = False, + llm_router: Router | None = None, + caller_aliases: CallerAliases = GLOBAL_ALIASES_ONLY, ) -> dict[str, Any]: """ Enrich a model dictionary with litellm model info (pricing, context window, etc.) @@ -16129,6 +16117,7 @@ def _enrich_model_info_with_litellm_data( model: Model dictionary to enrich debug: Whether to include debug information like openai_client llm_router: Optional router instance for debug info + caller_aliases: The caller's alias maps, applied before deciding which model a wildcard row prices as Returns: Enriched model dictionary with sensitive info removed @@ -16146,15 +16135,22 @@ def _enrich_model_info_with_litellm_data( # read litellm model_prices_and_context_window.json to get the following: # input_cost_per_token, output_cost_per_token, max_tokens - priced_as: Final = wildcard_row_priced_model(model, llm_router) + priced_as: Final = wildcard_row_priced_model(model, llm_router, caller_aliases) priced_model: Final = ( - model if priced_as is None else {**model, "litellm_params": {**model["litellm_params"], "model": priced_as}} + {**model, "litellm_params": {**model["litellm_params"], "model": priced_as}} + if isinstance(priced_as, str) + else model ) priced_litellm_model: Final = priced_model.get("litellm_params", {}).get("model") - litellm_model_info: Final = get_litellm_model_info(model=priced_model) or ( - model_info_cost_map_entry(priced_litellm_model) - if isinstance(priced_litellm_model, str) - else MappingProxyType({}) + litellm_model_info: Final = ( + MappingProxyType({}) + if isinstance(priced_as, RoutedElsewhere) + else get_litellm_model_info(model=priced_model) + or ( + model_info_cost_map_entry(priced_litellm_model) + if isinstance(priced_litellm_model, str) + else MappingProxyType({}) + ) ) discovered_model_info: Final = ( llm_router.registry().discovered_model_info(model_info.get("id")) @@ -17749,8 +17745,23 @@ def _translate_model_name_for_response(model: dict) -> dict[str, object]: return {**model, "model_name": team_public} -def _get_proxy_model_info(model: dict) -> dict: - return _translate_model_name_for_response(_enrich_model_info_with_litellm_data(model=model, llm_router=llm_router)) +def _get_proxy_model_info(model: dict, caller_aliases: CallerAliases = GLOBAL_ALIASES_ONLY) -> dict: + return _translate_model_name_for_response( + _enrich_model_info_with_litellm_data(model=model, llm_router=llm_router, caller_aliases=caller_aliases) + ) + + +async def request_caller_aliases(user_api_key_dict: UserAPIKeyAuth, listed_team_id: str | None = None) -> CallerAliases: + router_settings: Final = await proxy_config.get_hierarchical_router_settings( + user_api_key_dict, prisma_client, proxy_logging_obj + ) + return caller_alias_maps( + user_api_key_dict.aliases, + user_api_key_dict.team_model_aliases, + user_api_key_dict.team_id, + listed_team_id, + router_settings.get("model_group_alias") if isinstance(router_settings, Mapping) else None, + ) def _model_info_json_response(data: Sequence[Mapping[str, object]] | Mapping[str, object]) -> Response: @@ -17887,7 +17898,10 @@ async def model_info_v1( status_code=400, detail={"error": f"Model id = {litellm_model_id} not found on litellm proxy"}, ) - _deployment_info_dict = _get_proxy_model_info(model=deployment_info.model_dump(exclude_none=True)) + _deployment_info_dict = _get_proxy_model_info( + model=deployment_info.model_dump(exclude_none=True), + caller_aliases=await request_caller_aliases(user_api_key_dict), + ) single_model_list: list[dict] = [_deployment_info_dict] if prisma_client is not None: single_model_list = await _populate_team_access_on_models( @@ -17956,8 +17970,11 @@ async def model_info_v1( if include_team_models: all_models = _filter_models_to_user_accessible(all_models) + caller_aliases: Final = await request_caller_aliases(user_api_key_dict) all_models = [ - _translate_model_name_for_response(_enrich_model_info_with_litellm_data(model=model, llm_router=llm_router)) + _translate_model_name_for_response( + _enrich_model_info_with_litellm_data(model=model, llm_router=llm_router, caller_aliases=caller_aliases) + ) for model in all_models ] diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index aaaee87c3fa..1125bc0b005 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -4494,6 +4494,47 @@ class TestModelInfoCostMapEchoFilter: assert info["access_groups"] == ["prod"] assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto the whole wildcard" + @pytest.mark.asyncio + async def test_echo_of_a_team_wildcard_row_is_not_persisted_on_the_wildcard(self): + import litellm + from litellm.proxy.management_endpoints.model_management_endpoints import _update_team_model_in_db + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + router = litellm.Router( + model_list=[ + { + "model_name": "team-openai/*", + "litellm_params": {"model": "openai/*", "api_key": "x"}, + "model_info": {"id": "dep-team-wildcard", "team_id": "team-a"}, + } + ] + ) + entry = litellm.get_model_info("openai/gpt-5.6") + echo = {**entry, "id": "dep-team-wildcard", "team_id": "team-a", "access_groups": ["prod"]} + db_model = Deployment( + model_name="team-openai/*", + litellm_params=LiteLLM_Params(model="openai/*"), + model_info=ModelInfo(id="dep-team-wildcard", team_id="team-a"), + ) + + with ( + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.management_endpoints.model_management_endpoints.team_model_add"), + patch("litellm.proxy.management_endpoints.model_management_endpoints.team_model_delete"), + ): + result = await _update_team_model_in_db( + db_model=db_model, + patch_data=updateDeployment(model_name="team-openai/gpt-5.6", model_info=ModelInfo(**echo)), + user_api_key_dict=UserAPIKeyAuth(user_id="test_user", user_role=LitellmUserRoles.PROXY_ADMIN), + prisma_client=MockPrismaClient(team_exists=True), + write_row=_passthrough_row, + llm_router=router, + ) + + info = json.loads(result["model_info"]) + assert info["access_groups"] == ["prod"] + assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto a team's wildcard" + def test_an_edited_value_survives_the_echo_filter(self): import litellm from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 5c900a6eb5b..a20b9f3572e 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -314,7 +314,7 @@ def configured_router(monkeypatch): monkeypatch.setattr(proxy_server, "llm_router", router) monkeypatch.setattr(proxy_server, "llm_model_list", [{"model_name": "gpt-4"}]) monkeypatch.setattr(proxy_server, "user_model", None) - monkeypatch.setattr(proxy_server, "_get_proxy_model_info", lambda model: model) + monkeypatch.setattr(proxy_server, "_get_proxy_model_info", lambda model, caller_aliases: model) yield router @@ -678,9 +678,9 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( ) -> None: """``acme-lite`` has a deployment of its own, ``acme-pro`` is an alias of it, ``acme-max`` is a routing group over it, and ``my-openai/gpt-acme-*`` is a narrower wildcard, so calls to those names - never reach the catch-all or ``my-openai/*``, which still list them. Such a row shows no price, even - where another provider's model goes by the same name, and every price shown is the one calls are - billed at.""" + never reach the catch-all or ``my-openai/*``, which still list them. Such a row shows no price or + provider, even where another provider's model goes by the same name, and every price shown is the one + calls are billed at.""" from litellm.proxy.auth import model_checks catalog: Final = { @@ -791,6 +791,23 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( ("fireworks-wildcard", "acme-pro"), ("openai-wildcard", "my-openai/gpt-acme-lite"), ] + shown_models: Final = { + (row["model_info"]["id"], row["model_name"]): ( + row["model_info"].get("litellm_provider"), + row["model_info"].get("key"), + ) + for row in response.json()["data"] + if "*" not in row["model_name"] and row["model_info"]["id"] != "deepseek-acme-lite" + } + assert shown_models == dict.fromkeys( + [ + ("fireworks-wildcard", "acme-lite"), + ("fireworks-wildcard", "acme-max"), + ("fireworks-wildcard", "acme-pro"), + ("openai-wildcard", "my-openai/gpt-acme-lite"), + ], + (None, None), + ) assert listed[("deepseek-acme-lite", "acme-lite")] == (7e-07, 8e-07) assert {key: prices for key, prices in listed.items() if key[0] != "deepseek-acme-lite"} == { ("fireworks-wildcard", "acme-lite"): (None, None), @@ -1023,6 +1040,42 @@ def test_model_info_prices_a_wildcard_row_aliased_to_a_sibling_name_as_the_alias _invalidate_model_cost_lowercase_map() +def test_model_info_prices_a_wildcard_row_the_proxy_renames_as_the_model_its_calls_reach( + client: TestClient, + auth_as: Callable[..., AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """The proxy renames a requested model through the global ``model_alias_map``, the key's ``aliases`` and the + key's ``router_settings.model_group_alias`` before the router sees it. A row renamed to another deployment + shows no price or provider, and a row renamed to a sibling name of its own wildcard shows that sibling's + price.""" + _catch_all_and_shadowed_wildcard_router(monkeypatch) + monkeypatch.setattr(litellm, "model_alias_map", {"my-fw/acme-max": "acme-lite"}) + + with auth_as( + aliases={"openai/gpt-5.6-acme": "acme-lite"}, + router_settings={"model_group_alias": {"my-fw/acme-pro": "my-fw/acme-mini"}}, + ): + rows: Final = _expanded_wildcard_rows(client) + + shown: Final = { + name: ( + row["model_info"].get("input_cost_per_token"), + row["model_info"].get("output_cost_per_token"), + row["model_info"].get("litellm_provider"), + ) + for name, row in rows.items() + } + assert shown == { + "my-fw/acme-max": (None, None, None), + "my-fw/acme-mini": (5e-08, 6e-08, "fireworks_ai"), + "my-fw/acme-pro": (5e-08, 6e-08, "fireworks_ai"), + "openai/gpt-5.6-acme": (None, None, None), + } + _invalidate_model_cost_lowercase_map() + + def test_saving_a_replayed_model_info_wildcard_row_stores_none_of_its_shown_metadata( client: TestClient, auth_as: Callable[[], AbstractContextManager[object]], @@ -1031,7 +1084,8 @@ def test_saving_a_replayed_model_info_wildcard_row_stores_none_of_its_shown_meta ) -> None: """An API client that saves a ``/model/info`` row back unchanged, apart from one edit, must not pin the cost-map metadata the row showed onto the wildcard, whether the row is a catch-all name carrying its - provider, a name an alias sends to another deployment, or a name an alias sends to a sibling name.""" + provider or a name an alias sends to a sibling name. A name an alias sends to another deployment shows + no metadata to pin.""" from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment @@ -1062,7 +1116,11 @@ def test_saving_a_replayed_model_info_wildcard_row_stores_none_of_its_shown_meta shown_metadata: Final = { name: set(row["model_info"]) - set(sources[row["model_info"]["id"]]["model_info"]) for name, row in rows.items() } - assert all(shown_metadata.values()), shown_metadata + assert sorted(name for name, row in rows.items() if row["model_info"].get("key")) == [ + "my-fw/acme-max", + "my-fw/acme-mini", + "openai/gpt-5.6-acme", + ] assert {name: info["access_groups"] for name, info in stored.items()} == dict.fromkeys(rows, ["prod"]) assert {name: shown_metadata[name] & set(info) for name, info in stored.items()} == dict.fromkeys(rows, set()) _invalidate_model_cost_lowercase_map() diff --git a/tests/unit/proxy/proxy_server/test_team_model_name_translation.py b/tests/unit/proxy/proxy_server/test_team_model_name_translation.py index b647fbdd02a..e2b4acc0c09 100644 --- a/tests/unit/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/unit/proxy/proxy_server/test_team_model_name_translation.py @@ -808,7 +808,7 @@ async def test_model_info_v1_litellm_model_id_include_team_models_filters_inacce monkeypatch.setattr(ps, "llm_model_list", [team_row]) monkeypatch.setattr(ps, "llm_router", router) monkeypatch.setattr(ps, "prisma_client", MagicMock()) - monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model: team_row) + monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model, caller_aliases: team_row) monkeypatch.setattr(ps, "_populate_team_access_on_models", _fake_populate) caller = UserAPIKeyAuth( @@ -843,7 +843,7 @@ async def test_model_info_v1_litellm_model_id_team_id_applies_team_filter(monkey monkeypatch.setattr(ps, "llm_model_list", [team_row]) monkeypatch.setattr(ps, "llm_router", router) monkeypatch.setattr(ps, "prisma_client", MagicMock()) - monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model: team_row) + monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model, caller_aliases: team_row) monkeypatch.setattr(ps, "_populate_team_access_on_models", _fake_populate) monkeypatch.setattr(ps, "_filter_models_by_team_id", team_filter) From e5ad3044c76bda30a6ec6c1338dbb732635c7c39 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 01:36:21 -0700 Subject: [PATCH 08/12] fix(proxy): keep same-model wildcard row prices, skip deleted team aliases, use the caller's team --- litellm/proxy/auth/model_checks.py | 77 +++++++-- .../proxy/common_utils/model_listing_utils.py | 7 +- .../model_management_endpoints.py | 21 +-- .../proxy_server/test_routes_model_info.py | 146 +++++++++++++++++- 4 files changed, 221 insertions(+), 30 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 331dfc4b19c..ac218507fc5 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -5,7 +5,7 @@ import functools import itertools import re from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from types import MappingProxyType from typing import Final @@ -14,7 +14,7 @@ from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth -from litellm.proxy.common_utils.model_listing_utils import CallerAliases, served_model +from litellm.proxy.common_utils.model_listing_utils import CallerAliases, alias_map, served_model from litellm.proxy.common_utils.token_count_settings import cost_map_entry from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.router import Router @@ -594,20 +594,69 @@ def wildcard_row_priced_model( source_model: Final = _litellm_model(source) if not isinstance(source_pattern, str) or not isinstance(source_model, str) or _litellm_model(row) == source_model: return None - served_name: Final = served_model(listed_name, caller_aliases) - return wildcard_listed_priced_model( - source_pattern, - source_model, - routed_model_name(llm_router, served_name), - routes_through_wildcard=routes_through_deployment(llm_router, model_id, served_name, _deployment_team_id(row)), + return wildcard_deployment_priced_model( + llm_router, + WildcardDeployment(model_id, source_pattern, source_model, _deployment_team_id(row)), + listed_name, + caller_aliases, ) -def wildcard_listed_priced_model( - model_name_pattern: str, wildcard_model: str, routed_name: str, routes_through_wildcard: bool +@dataclass(frozen=True, slots=True) +class WildcardDeployment: + model_id: str + model_name_pattern: str + litellm_model: str + team_id: str | None + + +def wildcard_deployment_priced_model( + llm_router: Router, deployment: WildcardDeployment, listed_name: str, caller_aliases: CallerAliases ) -> str | RoutedElsewhere: - if not routes_through_wildcard: - return ROUTED_ELSEWHERE + registry: Final = llm_router.registry() + served_name: Final = served_model(listed_name, _without_deleted_team_targets(caller_aliases, registry)) + routed_name: Final = routed_model_name(llm_router, served_name) + priced: Final = wildcard_listed_priced_model(deployment.model_name_pattern, deployment.litellm_model, routed_name) + candidates: Final = registry.names( + served_name, kind="route_candidates", team_id=deployment.team_id or caller_aliases.team_id + ) + if deployment.model_id in candidates or _every_candidate_runs(registry, candidates, routed_name, priced): + return priced + return ROUTED_ELSEWHERE + + +def _every_candidate_runs(registry: RouterRegistry, candidates: frozenset[str], routed_name: str, model: str) -> bool: + return bool(candidates) and all( + _model_run_by(_router_deployment(registry, candidate), routed_name) == model for candidate in candidates + ) + + +def _model_run_by(deployment: Mapping[str, object] | None, routed_name: str) -> object: + if deployment is None: + return None + model_name: Final = deployment.get("model_name") + litellm_model: Final = _litellm_model(deployment) + if not isinstance(model_name, str) or not isinstance(litellm_model, str): + return litellm_model + return wildcard_listed_priced_model(model_name, litellm_model, routed_name) + + +def _without_deleted_team_targets(caller_aliases: CallerAliases, registry: RouterRegistry) -> CallerAliases: + deleted_prefix: Final = f"model_name_{caller_aliases.team_id}_" + live_names: Final = registry.index("model_name") + return replace( + caller_aliases, + team=MappingProxyType( + { + alias: target + for alias, target in alias_map(caller_aliases.team).items() + if not target.startswith(deleted_prefix) or target in live_names + } + ), + ) + + +def wildcard_listed_priced_model(model_name_pattern: str, wildcard_model: str, routed_name: str) -> str: filled: Final = wildcard_filled_model(model_name_pattern, wildcard_model, routed_name) if filled is None: return wildcard_model @@ -623,10 +672,6 @@ def routed_model_name(llm_router: Router, listed_name: str) -> str: return llm_router.registry().resolve(listed_name, kind="alias") or listed_name -def routes_through_deployment(llm_router: Router, model_id: str, listed_name: str, team_id: str | None) -> bool: - return model_id in llm_router.registry().names(listed_name, kind="route_candidates", team_id=team_id) - - def model_info_cost_map_entry(model: str) -> Mapping[str, object]: provider, _, _ = model.partition("/") return cost_map_entry(model) or cost_map_entry(model.rsplit("/", 1)[-1], custom_llm_provider=provider) diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py index 2f7d22afdd1..44ee6cd7a0d 100644 --- a/litellm/proxy/common_utils/model_listing_utils.py +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -160,8 +160,8 @@ class ClaudeCodeRoutingNames: @dataclass(frozen=True, slots=True) class CallerAliases: - """The caller's `team` and `key` alias maps, the names `/v1/models` lists for it, and `router`, - its key's or team's `router_settings.model_group_alias`. + """The caller's `team` and `key` alias maps, the names `/v1/models` lists for it, `router`, + its key's or team's `router_settings.model_group_alias`, and `team_id`, the key's team. `rewrite` are the maps `/chat/completions` rewrites its model through after auth's `router` rewrite, in the order it applies them: the team's, the key's in `add_litellm_data_to_request`, then the global `model_alias_map` and the key's again in `common_processing_pre_call_logic`, @@ -170,6 +170,7 @@ class CallerAliases: team: object key: object router: object = None + team_id: str | None = None @property def own(self) -> tuple[object, ...]: @@ -214,7 +215,7 @@ def caller_alias_maps( """Team aliases count only when listing the team the key authenticated as.""" if listed_team_id is not None and listed_team_id != key_team_id: return CallerAliases(None, key_aliases, router_aliases) - return CallerAliases(team_aliases, key_aliases, router_aliases) + return CallerAliases(team_aliases, key_aliases, router_aliases, key_team_id) def alias_map(aliases: object) -> Mapping[str, str]: diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 774f1216084..f9e7e67aece 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -63,9 +63,9 @@ from litellm.proxy._types import ( from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY from litellm.proxy.auth.model_checks import ( GLOBAL_ALIASES_ONLY, + WildcardDeployment, model_info_cost_map_entry, - routed_model_name, - routes_through_deployment, + wildcard_deployment_priced_model, wildcard_listed_priced_model, ) from litellm.proxy.auth.team_grants import team_model_aliases @@ -1280,14 +1280,15 @@ def _live_cost_map_model( if stored_model is None or listed_name is None: return stored_model model_id: Final = db_model.model_info.id - served_name: Final = served_model(listed_name, caller_aliases) - priced: Final = wildcard_listed_priced_model( - db_model.model_name, - stored_model, - served_name if llm_router is None else routed_model_name(llm_router, served_name), - routes_through_wildcard=llm_router is None - or model_id is None - or routes_through_deployment(llm_router, model_id, served_name, db_model.model_info.team_id), + if llm_router is None or model_id is None: + return wildcard_listed_priced_model( + db_model.model_name, stored_model, served_model(listed_name, caller_aliases) + ) + priced: Final = wildcard_deployment_priced_model( + llm_router, + WildcardDeployment(model_id, db_model.model_name, stored_model, db_model.model_info.team_id), + listed_name, + caller_aliases, ) return priced if isinstance(priced, str) else stored_model diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index a20b9f3572e..26ee2cb214f 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -25,7 +25,7 @@ import litellm from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy import proxy_server -from litellm.proxy._types import LiteLLM_ProxyModelTable +from litellm.proxy._types import LiteLLM_ProxyModelTable, UserAPIKeyAuth from litellm.utils import _invalidate_model_cost_lowercase_map from .conftest import normalize, route_entry_points_to_stubs # type: ignore[import-not-found] @@ -1076,6 +1076,150 @@ def test_model_info_prices_a_wildcard_row_the_proxy_renames_as_the_model_its_cal _invalidate_model_cost_lowercase_map() +def test_model_info_prices_a_wildcard_row_whose_team_alias_targets_a_deleted_team_deployment( + client: TestClient, + auth_as: Callable[..., AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """Calls skip a team alias whose target team deployment was deleted, so the row that alias names keeps the + price its calls through the wildcard are billed at.""" + from litellm.proxy.litellm_pre_call_utils import _update_model_if_team_alias_exists + + _catch_all_and_shadowed_wildcard_router(monkeypatch) + caller: Final = {"team_id": "team-a", "team_model_aliases": {"my-fw/acme-mini": "model_name_team-a_deleted"}} + + with auth_as(**caller): + rows: Final = _expanded_wildcard_rows(client) + + request: Final = {"model": "my-fw/acme-mini"} + _update_model_if_team_alias_exists(data=request, user_api_key_dict=UserAPIKeyAuth(**caller)) + shown: Final = rows["my-fw/acme-mini"]["model_info"] + assert request["model"] == "my-fw/acme-mini" + assert ( + shown.get("input_cost_per_token"), + shown.get("output_cost_per_token"), + shown.get("litellm_provider"), + ) == (5e-08, 6e-08, "fireworks_ai") + _invalidate_model_cost_lowercase_map() + + +def _fireworks_wildcard_rows(client: TestClient) -> dict[str, dict[str, object]]: + response: Final = client.get("/model/info") + assert response.status_code == 200, response.text + return { + row["model_name"]: row["model_info"] + for row in response.json()["data"] + if row["model_info"]["id"] == "fireworks-wildcard" and "*" not in row["model_name"] + } + + +_FIREWORKS_WILDCARD: Final = { + "model_name": "my-fw/*", + "litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/*", "api_key": "x"}, + "model_info": {"id": "fireworks-wildcard"}, +} + + +def test_model_info_keeps_the_price_of_a_wildcard_row_another_deployment_of_the_same_model_serves( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """A deployment pinned to one model a wildcard lists takes that name's calls but runs the model the wildcard + would run, so the wildcard's row keeps the price those calls bill, and saving that row back stores none of it. + A name pinned to another model shows no price.""" + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment + + _catch_all_and_shadowed_wildcard_router(monkeypatch) + router: Final = litellm.Router( + model_list=[ + _FIREWORKS_WILDCARD, + { + "model_name": "my-fw/acme-mini", + "litellm_params": { + "model": "fireworks_ai/accounts/fireworks/models/acme-mini", + "api_key": "x", + "rpm": 10, + }, + "model_info": {"id": "pinned-acme-mini"}, + }, + { + "model_name": "my-fw/acme-pro", + "litellm_params": {"model": "deepseek/acme-lite", "api_key": "x"}, + "model_info": {"id": "pinned-acme-pro"}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + + with auth_as(): + rows: Final = _fireworks_wildcard_rows(client) + + shown: Final = rows["my-fw/acme-mini"] + call: Final = router.completion( + model="my-fw/acme-mini", messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + stored: Final = json.loads( + update_db_model( + db_model=Deployment( + model_name="my-fw/*", + litellm_params=LiteLLM_Params(model="fireworks_ai/accounts/fireworks/models/*"), + model_info=ModelInfo(id="fireworks-wildcard"), + ), + updated_patch=updateDeployment( + model_name="my-fw/acme-mini", model_info=ModelInfo(**{**shown, "access_groups": ["prod"]}) + ), + llm_router=router, + )["model_info"] + ) + assert call._hidden_params["model_id"] == "pinned-acme-mini" + assert call._hidden_params["response_cost"] == pytest.approx( + shown["input_cost_per_token"] * call.usage.prompt_tokens + + shown["output_cost_per_token"] * call.usage.completion_tokens + ) + assert stored["access_groups"] == ["prod"] + assert set(stored).isdisjoint(set(shown) - set(router.registry().deployment_dict("fireworks-wildcard")["model_info"])) + assert rows["my-fw/acme-pro"].get("input_cost_per_token") is None + _invalidate_model_cost_lowercase_map() + + +def test_model_info_shows_no_wildcard_price_for_a_name_the_callers_team_publishes( + client: TestClient, + auth_as: Callable[..., AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, +) -> None: + """Team ``team-a`` publishes ``my-fw/acme-mini`` on its own DeepSeek deployment, so a team-a key's calls to that + name skip the global wildcard and the wildcard's row shows that key no price, while a key outside the team + still sees the Fireworks price its calls bill.""" + _catch_all_and_shadowed_wildcard_router(monkeypatch) + router: Final = litellm.Router( + model_list=[ + _FIREWORKS_WILDCARD, + { + "model_name": "model_name_team-a_acme-mini", + "litellm_params": {"model": "deepseek/acme-lite", "api_key": "x"}, + "model_info": {"id": "team-deepseek", "team_id": "team-a", "team_public_model_name": "my-fw/acme-mini"}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + + with auth_as(team_id="team-a"): + team_rows: Final = _fireworks_wildcard_rows(client) + with auth_as(): + other_rows: Final = _fireworks_wildcard_rows(client) + + assert team_rows["my-fw/acme-mini"].get("input_cost_per_token") is None + assert other_rows["my-fw/acme-mini"].get("input_cost_per_token") == 5e-08 + _invalidate_model_cost_lowercase_map() + + def test_saving_a_replayed_model_info_wildcard_row_stores_none_of_its_shown_metadata( client: TestClient, auth_as: Callable[[], AbstractContextManager[object]], From d3c511ef93af65df4e18dbd9d2212101bd77ab8b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 02:39:25 -0700 Subject: [PATCH 09/12] fix(proxy): keep live team aliases when skipping deleted team alias targets --- litellm/proxy/auth/model_checks.py | 12 +++++------- .../proxy_server/test_routes_model_info.py | 17 ++++++++++++----- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index ac218507fc5..71848adc094 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -646,13 +646,11 @@ def _without_deleted_team_targets(caller_aliases: CallerAliases, registry: Route live_names: Final = registry.index("model_name") return replace( caller_aliases, - team=MappingProxyType( - { - alias: target - for alias, target in alias_map(caller_aliases.team).items() - if not target.startswith(deleted_prefix) or target in live_names - } - ), + team={ + alias: target + for alias, target in alias_map(caller_aliases.team).items() + if not target.startswith(deleted_prefix) or target in live_names + }, ) diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 26ee2cb214f..064c4574a23 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -1083,24 +1083,31 @@ def test_model_info_prices_a_wildcard_row_whose_team_alias_targets_a_deleted_tea local_model_cost_map: None, ) -> None: """Calls skip a team alias whose target team deployment was deleted, so the row that alias names keeps the - price its calls through the wildcard are billed at.""" + price its calls through the wildcard are billed at, while the team's live aliases still apply.""" from litellm.proxy.litellm_pre_call_utils import _update_model_if_team_alias_exists _catch_all_and_shadowed_wildcard_router(monkeypatch) - caller: Final = {"team_id": "team-a", "team_model_aliases": {"my-fw/acme-mini": "model_name_team-a_deleted"}} + caller: Final = { + "team_id": "team-a", + "team_model_aliases": {"my-fw/acme-mini": "model_name_team-a_deleted", "openai/gpt-5.6-acme": "my-fw/acme-mini"}, + } with auth_as(**caller): rows: Final = _expanded_wildcard_rows(client) - request: Final = {"model": "my-fw/acme-mini"} - _update_model_if_team_alias_exists(data=request, user_api_key_dict=UserAPIKeyAuth(**caller)) + stale_request: Final = {"model": "my-fw/acme-mini"} + live_request: Final = {"model": "openai/gpt-5.6-acme"} + _update_model_if_team_alias_exists(data=stale_request, user_api_key_dict=UserAPIKeyAuth(**caller)) + _update_model_if_team_alias_exists(data=live_request, user_api_key_dict=UserAPIKeyAuth(**caller)) shown: Final = rows["my-fw/acme-mini"]["model_info"] - assert request["model"] == "my-fw/acme-mini" + live_alias_row: Final = rows["openai/gpt-5.6-acme"]["model_info"] + assert (stale_request["model"], live_request["model"]) == ("my-fw/acme-mini", "my-fw/acme-mini") assert ( shown.get("input_cost_per_token"), shown.get("output_cost_per_token"), shown.get("litellm_provider"), ) == (5e-08, 6e-08, "fireworks_ai") + assert (live_alias_row.get("input_cost_per_token"), live_alias_row.get("litellm_provider")) == (None, None) _invalidate_model_cost_lowercase_map() From b7878751f7868ccdb61cb2b486bc24b9bb0ded37 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 03:58:58 -0700 Subject: [PATCH 10/12] fix(proxy): show no wildcard row price when the same-model deployment serving it sets custom rates --- litellm/proxy/auth/model_checks.py | 36 ++++++++++++++----- .../proxy_server/test_routes_model_info.py | 22 +++++++++++- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 71848adc094..9021929777b 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -22,7 +22,7 @@ from litellm.router_api import RouterRegistry from litellm.router_utils.fallback_event_handlers import get_fallback_model_group from litellm.router_utils.pattern_match_deployments import PatternMatchRouter from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params -from litellm.types.utils import LlmProviders +from litellm.types.utils import CustomPricingLiteLLMParams, LlmProviders from litellm.utils import ProviderConfigManager, get_valid_models LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders) @@ -620,25 +620,45 @@ def wildcard_deployment_priced_model( candidates: Final = registry.names( served_name, kind="route_candidates", team_id=deployment.team_id or caller_aliases.team_id ) - if deployment.model_id in candidates or _every_candidate_runs(registry, candidates, routed_name, priced): + wildcard_run: Final = (priced, _pricing_overrides(_router_deployment(registry, deployment.model_id))) + if deployment.model_id in candidates or _every_candidate_runs(registry, candidates, routed_name, wildcard_run): return priced return ROUTED_ELSEWHERE -def _every_candidate_runs(registry: RouterRegistry, candidates: frozenset[str], routed_name: str, model: str) -> bool: +_PRICING_FIELDS: Final = frozenset((*CustomPricingLiteLLMParams.model_fields, "base_model")) + + +def _every_candidate_runs( + registry: RouterRegistry, candidates: frozenset[str], routed_name: str, run: tuple[object, Mapping[str, object]] +) -> bool: return bool(candidates) and all( - _model_run_by(_router_deployment(registry, candidate), routed_name) == model for candidate in candidates + _run_of(_router_deployment(registry, candidate), routed_name) == run for candidate in candidates ) -def _model_run_by(deployment: Mapping[str, object] | None, routed_name: str) -> object: +def _run_of(deployment: Mapping[str, object] | None, routed_name: str) -> tuple[object, Mapping[str, object]] | None: if deployment is None: return None model_name: Final = deployment.get("model_name") litellm_model: Final = _litellm_model(deployment) - if not isinstance(model_name, str) or not isinstance(litellm_model, str): - return litellm_model - return wildcard_listed_priced_model(model_name, litellm_model, routed_name) + model: Final = ( + wildcard_listed_priced_model(model_name, litellm_model, routed_name) + if isinstance(model_name, str) and isinstance(litellm_model, str) + else litellm_model + ) + return (model, _pricing_overrides(deployment)) + + +def _pricing_overrides(deployment: Mapping[str, object] | None) -> Mapping[str, object]: + blobs: Final = () if deployment is None else (deployment.get("model_info"), deployment.get("litellm_params")) + return MappingProxyType(dict(itertools.chain.from_iterable(_pricing_items(blob) for blob in blobs))) + + +def _pricing_items(blob: object) -> tuple[tuple[str, object], ...]: + if not is_str_keyed_mapping(blob): + return () + return tuple((key, value) for key, value in blob.items() if key in _PRICING_FIELDS and value is not None) def _without_deleted_team_targets(caller_aliases: CallerAliases, registry: RouterRegistry) -> CallerAliases: diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 064c4574a23..d98603da30c 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -1136,7 +1136,7 @@ def test_model_info_keeps_the_price_of_a_wildcard_row_another_deployment_of_the_ ) -> None: """A deployment pinned to one model a wildcard lists takes that name's calls but runs the model the wildcard would run, so the wildcard's row keeps the price those calls bill, and saving that row back stores none of it. - A name pinned to another model shows no price.""" + A name pinned to another model, or to the same model at custom rates, shows no price.""" from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment @@ -1158,6 +1158,16 @@ def test_model_info_keeps_the_price_of_a_wildcard_row_another_deployment_of_the_ "litellm_params": {"model": "deepseek/acme-lite", "api_key": "x"}, "model_info": {"id": "pinned-acme-pro"}, }, + { + "model_name": "my-fw/acme-max", + "litellm_params": { + "model": "fireworks_ai/accounts/fireworks/models/acme-max", + "api_key": "x", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 3e-06, + }, + "model_info": {"id": "pinned-acme-max"}, + }, ] ) monkeypatch.setattr(proxy_server, "llm_router", router) @@ -1191,6 +1201,16 @@ def test_model_info_keeps_the_price_of_a_wildcard_row_another_deployment_of_the_ assert stored["access_groups"] == ["prod"] assert set(stored).isdisjoint(set(shown) - set(router.registry().deployment_dict("fireworks-wildcard")["model_info"])) assert rows["my-fw/acme-pro"].get("input_cost_per_token") is None + custom_rate_call: Final = router.completion( + model="my-fw/acme-max", messages=[{"role": "user", "content": "hi"}], mock_response="hello there" + ) + assert custom_rate_call._hidden_params["response_cost"] == pytest.approx( + 2e-06 * custom_rate_call.usage.prompt_tokens + 3e-06 * custom_rate_call.usage.completion_tokens + ) + assert (rows["my-fw/acme-max"].get("input_cost_per_token"), rows["my-fw/acme-max"].get("litellm_provider")) == ( + None, + None, + ) _invalidate_model_cost_lowercase_map() From 9128a71f493e44292f298f3fadb9d47517653113 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 14:15:56 -0700 Subject: [PATCH 11/12] fix(model_info): recognize a replayed wildcard row on a worker that has not loaded the wildcard --- litellm/proxy/auth/model_checks.py | 16 ++++++-- .../model_management_endpoints.py | 14 ++----- .../test_model_management_endpoints.py | 39 ++++++++++++++++--- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index d1e6e8eed7f..ac4da828bb6 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -685,9 +685,8 @@ class WildcardDeployment: def wildcard_deployment_priced_model( llm_router: Router, deployment: WildcardDeployment, listed_name: str, caller_aliases: CallerAliases ) -> str | RoutedElsewhere: - registry: Final = llm_router.registry() - served_name: Final = dispatched_model(listed_name, _without_deleted_team_targets(caller_aliases, registry)) - candidates: Final = registry.names( + served_name: Final = _served_name(llm_router, listed_name, caller_aliases) + candidates: Final = llm_router.registry().names( served_name, kind="route_candidates", team_id=deployment.team_id or caller_aliases.team_id ) if deployment.model_id not in candidates: @@ -697,6 +696,17 @@ def wildcard_deployment_priced_model( ) +def wildcard_row_filled_model( + llm_router: Router, name_pattern: str, litellm_model: str, listed_name: str, caller_aliases: CallerAliases +) -> str: + served_name: Final = _served_name(llm_router, listed_name, caller_aliases) + return _routed_deployment_model(name_pattern, routed_model_name(llm_router, served_name), litellm_model) + + +def _served_name(llm_router: Router, listed_name: str, caller_aliases: CallerAliases) -> str: + return dispatched_model(listed_name, _without_deleted_team_targets(caller_aliases, llm_router.registry())) + + def _without_deleted_team_targets(caller_aliases: CallerAliases, registry: RouterRegistry) -> CallerAliases: deleted_prefix: Final = f"model_name_{caller_aliases.team_id}_" live_names: Final = registry.index("model_name") diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index f94ae658775..a51572a2888 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -63,9 +63,8 @@ from litellm.proxy._types import ( from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY from litellm.proxy.auth.model_checks import ( GLOBAL_ALIASES_ONLY, - WildcardDeployment, model_info_cost_map_entry, - wildcard_deployment_priced_model, + wildcard_row_filled_model, ) from litellm.proxy.auth.team_grants import team_model_aliases from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -1306,16 +1305,11 @@ _NO_LISTED_ROW: Final = _ListedRow() def _live_cost_map_model(db_model: Deployment, listed_row: _ListedRow) -> str | None: stored_model: Final = _decrypted_model(db_model.litellm_params.model) - model_id: Final = db_model.model_info.id - if stored_model is None or listed_row.name is None or listed_row.llm_router is None or model_id is None: + if stored_model is None or "*" not in stored_model or listed_row.name is None or listed_row.llm_router is None: return stored_model - priced: Final = wildcard_deployment_priced_model( - listed_row.llm_router, - WildcardDeployment(model_id, db_model.model_name, stored_model, db_model.model_info.team_id), - listed_row.name, - listed_row.caller_aliases, + return wildcard_row_filled_model( + listed_row.llm_router, db_model.model_name, stored_model, listed_row.name, listed_row.caller_aliases ) - return priced if isinstance(priced, str) else stored_model def _cost_map_entry( diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 85475eb1fec..eb585e23f39 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -4845,6 +4845,40 @@ class TestModelInfoCostMapEchoFilter: assert info["access_groups"] == ["prod"] assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto the whole wildcard" + def test_echo_saved_on_a_worker_that_has_not_loaded_the_wildcard_is_not_persisted(self): + import litellm + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + router_without_the_wildcard = litellm.Router( + model_list=[ + { + "model_name": "openai-mini", + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "x"}, + "model_info": {"id": "dep-other"}, + } + ] + ) + entry = litellm.get_model_info("openai/gpt-5.6") + db_model = Deployment( + model_name="my-openai/*", + litellm_params=LiteLLM_Params(model="openai/*"), + model_info=ModelInfo(id="dep-echo-wildcard"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment( + model_name="my-openai/gpt-5.6", + model_info=ModelInfo(**{**entry, "id": "dep-echo-wildcard", "access_groups": ["prod"]}), + ), + llm_router=router_without_the_wildcard, + ) + + info = json.loads(result["model_info"]) + assert info["access_groups"] == ["prod"] + assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto the whole wildcard" + def test_legacy_update_echo_of_an_expanded_wildcard_row_is_not_persisted_on_the_wildcard(self): import litellm from litellm.proxy.management_endpoints.model_management_endpoints import ( @@ -8681,7 +8715,6 @@ class TestAccessGroupModelSync: mock_prisma = self._prisma_with_row("m-rename", "gpt-5.6", deployment_count=0) router = MagicMock() router.registry.return_value.names.return_value = ["m-rename"] - router.registry.return_value.resolve.return_value = None with self._endpoint_env(mock_prisma, router) as invalidate: await patch_model( @@ -8704,7 +8737,6 @@ class TestAccessGroupModelSync: mock_prisma = self._prisma_with_row("m-rename", "gpt-5.6", deployment_count=1) router = MagicMock() router.registry.return_value.names.return_value = ["m-rename"] - router.registry.return_value.resolve.return_value = None with self._endpoint_env(mock_prisma, router): await patch_model( @@ -8771,7 +8803,6 @@ class TestAccessGroupModelSync: mock_prisma = self._prisma_with_row("m-terraform", "gpt-5.6", deployment_count=0) router = MagicMock() router.registry.return_value.names.return_value = ["m-terraform"] - router.registry.return_value.resolve.return_value = None with self._endpoint_env(mock_prisma, router) as invalidate: await update_model( @@ -8798,7 +8829,6 @@ class TestAccessGroupModelSync: mock_prisma = self._prisma_with_row("m-rename", "gpt-5.6", deployment_count=0) router = MagicMock() router.registry.return_value.names.return_value = ["m-rename"] - router.registry.return_value.resolve.return_value = None evict = AsyncMock() with self._endpoint_env(mock_prisma, router, evict=evict): @@ -8844,7 +8874,6 @@ class TestAccessGroupModelSync: mock_prisma = self._prisma_with_row("m-rename", "gpt-5.6", deployment_count=1) router = MagicMock() router.registry.return_value.names.return_value = ["m-rename"] - router.registry.return_value.resolve.return_value = None with self._endpoint_env(mock_prisma, router): await patch_model( From 6de7d496166fbc501d23e10b479e6276277eeb57 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:28:52 -0700 Subject: [PATCH 12/12] test(proxy): feed wildcard discovery real provider data in model info tests --- .../test_model_management_endpoints.py | 42 +++++------ .../proxy_server/test_routes_model_info.py | 70 +++++++++---------- 2 files changed, 56 insertions(+), 56 deletions(-) diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index ea6cdc9c557..bb6700444bc 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -4865,7 +4865,7 @@ class TestModelInfoCostMapEchoFilter: from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo - router = litellm.Router( + router: Final = litellm.Router( model_list=[ { "model_name": "my-openai/*", @@ -4874,21 +4874,21 @@ class TestModelInfoCostMapEchoFilter: } ] ) - entry = litellm.get_model_info("openai/gpt-5.6") - echo = {**entry, "id": "dep-echo-wildcard", "db_model": True, "access_groups": ["prod"]} - db_model = Deployment( + entry: Final = litellm.get_model_info("openai/gpt-5.6") + echo: Final = {**entry, "id": "dep-echo-wildcard", "db_model": True, "access_groups": ["prod"]} + db_model: Final = Deployment( model_name="my-openai/*", litellm_params=LiteLLM_Params(model="openai/*"), model_info=ModelInfo(id="dep-echo-wildcard"), ) - result = update_db_model( + result: Final = update_db_model( db_model=db_model, updated_patch=updateDeployment(model_name="my-openai/gpt-5.6", model_info=ModelInfo(**echo)), llm_router=router, ) - info = json.loads(result["model_info"]) + info: Final = json.loads(result["model_info"]) assert info["access_groups"] == ["prod"] assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto the whole wildcard" @@ -4897,7 +4897,7 @@ class TestModelInfoCostMapEchoFilter: from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo - router_without_the_wildcard = litellm.Router( + router_without_the_wildcard: Final = litellm.Router( model_list=[ { "model_name": "openai-mini", @@ -4906,14 +4906,14 @@ class TestModelInfoCostMapEchoFilter: } ] ) - entry = litellm.get_model_info("openai/gpt-5.6") - db_model = Deployment( + entry: Final = litellm.get_model_info("openai/gpt-5.6") + db_model: Final = Deployment( model_name="my-openai/*", litellm_params=LiteLLM_Params(model="openai/*"), model_info=ModelInfo(id="dep-echo-wildcard"), ) - result = update_db_model( + result: Final = update_db_model( db_model=db_model, updated_patch=updateDeployment( model_name="my-openai/gpt-5.6", @@ -4922,7 +4922,7 @@ class TestModelInfoCostMapEchoFilter: llm_router=router_without_the_wildcard, ) - info = json.loads(result["model_info"]) + info: Final = json.loads(result["model_info"]) assert info["access_groups"] == ["prod"] assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto the whole wildcard" @@ -4934,7 +4934,7 @@ class TestModelInfoCostMapEchoFilter: ) from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo - router = litellm.Router( + router: Final = litellm.Router( model_list=[ { "model_name": "my-openai/*", @@ -4943,14 +4943,14 @@ class TestModelInfoCostMapEchoFilter: } ] ) - entry = litellm.get_model_info("openai/gpt-5.6") - db_model = Deployment( + entry: Final = litellm.get_model_info("openai/gpt-5.6") + db_model: Final = Deployment( model_name="my-openai/*", litellm_params=LiteLLM_Params(model="openai/*"), model_info=ModelInfo(id="dep-echo-wildcard"), ) - changes = legacy_update_model_info_changes( + changes: Final = legacy_update_model_info_changes( db_model, ModelInfo(**{**entry, "id": "dep-echo-wildcard", "access_groups": ["prod"]}), loaded_catalog=lambda: {}, @@ -4965,7 +4965,7 @@ class TestModelInfoCostMapEchoFilter: from litellm.proxy.management_endpoints.model_management_endpoints import _update_team_model_in_db from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo - router = litellm.Router( + router: Final = litellm.Router( model_list=[ { "model_name": "team-openai/*", @@ -4974,9 +4974,9 @@ class TestModelInfoCostMapEchoFilter: } ] ) - entry = litellm.get_model_info("openai/gpt-5.6") - echo = {**entry, "id": "dep-team-wildcard", "team_id": "team-a", "access_groups": ["prod"]} - db_model = Deployment( + entry: Final = litellm.get_model_info("openai/gpt-5.6") + echo: Final = {**entry, "id": "dep-team-wildcard", "team_id": "team-a", "access_groups": ["prod"]} + db_model: Final = Deployment( model_name="team-openai/*", litellm_params=LiteLLM_Params(model="openai/*"), model_info=ModelInfo(id="dep-team-wildcard", team_id="team-a"), @@ -4987,7 +4987,7 @@ class TestModelInfoCostMapEchoFilter: patch("litellm.proxy.management_endpoints.model_management_endpoints.team_model_add"), patch("litellm.proxy.management_endpoints.model_management_endpoints.team_model_delete"), ): - result = await _update_team_model_in_db( + result: Final = await _update_team_model_in_db( db_model=db_model, patch_data=updateDeployment(model_name="team-openai/gpt-5.6", model_info=ModelInfo(**echo)), user_api_key_dict=UserAPIKeyAuth(user_id="test_user", user_role=LitellmUserRoles.PROXY_ADMIN), @@ -4996,7 +4996,7 @@ class TestModelInfoCostMapEchoFilter: llm_router=router, ) - info = json.loads(result["model_info"]) + info: Final = json.loads(result["model_info"]) assert info["access_groups"] == ["prod"] assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto a team's wildcard" diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index b706dc328f3..19f0a245aca 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -12,7 +12,7 @@ from __future__ import annotations import asyncio import copy import json -from collections.abc import Callable +from collections.abc import Callable, Mapping from contextlib import AbstractContextManager from typing import Final from unittest.mock import AsyncMock, MagicMock @@ -539,6 +539,11 @@ def test_model_info_id_lookup_reports_the_same_cost_as_the_list( _invalidate_model_cost_lowercase_map() +def _discoverable_models(monkeypatch: pytest.MonkeyPatch, models: Mapping[str, list[str]]) -> None: + for provider, provider_models in models.items(): + monkeypatch.setitem(litellm.models_by_provider, provider, provider_models) + + async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monkeypatch): from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth from litellm.proxy.auth import model_checks @@ -585,7 +590,6 @@ def test_model_info_prices_expanded_wildcard_rows_as_the_model_their_calls_route """A catch-all over Fireworks lists ``acme-chat``, which also names a DeepSeek entry, and a prefixed OpenAI wildcard lists ``my-openai/gpt-acme``, which names no entry at all. Each row must show the provider and price its calls are billed at.""" - from litellm.proxy.auth import model_checks catalog: Final = { "fireworks_ai/accounts/fireworks/models/acme-chat": { @@ -609,14 +613,6 @@ def test_model_info_prices_expanded_wildcard_rows_as_the_model_their_calls_route } monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) _invalidate_model_cost_lowercase_map() - monkeypatch.setattr( - model_checks, - "get_provider_models", - lambda provider, litellm_params=None: { - "fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-chat"], - "openai": ["gpt-acme"], - }.get(provider, []), - ) router: Final = litellm.Router( model_list=[ { @@ -632,6 +628,13 @@ def test_model_info_prices_expanded_wildcard_rows_as_the_model_their_calls_route ] ) monkeypatch.setattr(proxy_server, "llm_router", router) + _discoverable_models( + monkeypatch, + { + "fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-chat"], + "openai": ["gpt-acme"], + }, + ) monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) monkeypatch.setattr(proxy_server, "user_model", None) @@ -678,7 +681,6 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( calls to those names never reach the catch-all, which still lists them. Such a row shows no price or provider, even where another provider's model goes by the same name, and every price shown is the one calls are billed at.""" - from litellm.proxy.auth import model_checks catalog: Final = { "fireworks_ai/accounts/fireworks/models/acme-lite": { @@ -720,17 +722,6 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( } monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) _invalidate_model_cost_lowercase_map() - monkeypatch.setattr( - model_checks, - "get_provider_models", - lambda provider, litellm_params=None: { - "fireworks_ai": [ - "fireworks_ai/accounts/fireworks/models/acme-lite", - "fireworks_ai/accounts/fireworks/models/acme-pro", - "fireworks_ai/accounts/fireworks/models/acme-max", - ], - }.get(provider, []), - ) router: Final = litellm.Router( model_list=[ { @@ -748,6 +739,16 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( routing_groups=[{"group_name": "acme-max", "models": ["acme-lite"], "routing_strategy": "simple-shuffle"}], ) monkeypatch.setattr(proxy_server, "llm_router", router) + _discoverable_models( + monkeypatch, + { + "fireworks_ai": [ + "fireworks_ai/accounts/fireworks/models/acme-lite", + "fireworks_ai/accounts/fireworks/models/acme-pro", + "fireworks_ai/accounts/fireworks/models/acme-max", + ], + }, + ) monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) monkeypatch.setattr(proxy_server, "user_model", None) @@ -808,7 +809,6 @@ def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip( def _catch_all_and_shadowed_wildcard_router(monkeypatch: pytest.MonkeyPatch) -> litellm.Router: - from litellm.proxy.auth import model_checks catalog: Final = { "gpt-5.6-acme": { @@ -844,18 +844,6 @@ def _catch_all_and_shadowed_wildcard_router(monkeypatch: pytest.MonkeyPatch) -> } monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog}) _invalidate_model_cost_lowercase_map() - monkeypatch.setattr( - model_checks, - "get_provider_models", - lambda provider, litellm_params=None: { - "openai": ["gpt-5.6-acme"], - "fireworks_ai": [ - "fireworks_ai/accounts/fireworks/models/acme-pro", - "fireworks_ai/accounts/fireworks/models/acme-max", - "fireworks_ai/accounts/fireworks/models/acme-mini", - ], - }.get(provider, []), - ) router: Final = litellm.Router( model_list=[ { @@ -877,6 +865,17 @@ def _catch_all_and_shadowed_wildcard_router(monkeypatch: pytest.MonkeyPatch) -> model_group_alias={"my-fw/acme-pro": "acme-lite", "my-fw/acme-max": "my-fw/acme-mini"}, ) monkeypatch.setattr(proxy_server, "llm_router", router) + _discoverable_models( + monkeypatch, + { + "openai": ["gpt-5.6-acme"], + "fireworks_ai": [ + "fireworks_ai/accounts/fireworks/models/acme-pro", + "fireworks_ai/accounts/fireworks/models/acme-max", + "fireworks_ai/accounts/fireworks/models/acme-mini", + ], + }, + ) monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) monkeypatch.setattr(proxy_server, "user_model", None) return router @@ -1025,6 +1024,7 @@ def test_model_info_shows_no_wildcard_price_for_a_name_the_callers_team_publishe ] ) monkeypatch.setattr(proxy_server, "llm_router", router) + _discoverable_models(monkeypatch, {"fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-mini"]}) monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) with auth_as(team_id="team-a"):