scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 280

fix(proxy): price expanded wildcard /model/info rows as the model their calls route to

Browse source
This commit is contained in:
mateo-berri 2026-10-29 06:57:28 -07:00
parent 67ae3fb527
commit d6954d57b5
5 changed files with 297 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
litellm/proxy/auth/model_checks.py
View file

@ -7,6 +7,8 @@ import re
from collections.abc import Callable, Mapping, Sequence
from typing import Final
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
@ -27,6 +29,8 @@ _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)
_CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)
_ROUTER_DEPLOYMENT: Final = TypeAdapter(object)
def _check_wildcard_routing(model: str) -> bool:
"""
@ -534,6 +538,57 @@ def expand_wildcard_deployments_for_model_info(
return expanded
def _litellm_model(deployment: Mapping[str, object]) -> object:
litellm_params: Final = deployment.get("litellm_params")
return litellm_params.get("model") if is_str_keyed_mapping(litellm_params) else None
def _router_deployment(llm_router: Router | None, model_id: object) -> Mapping[str, object] | None:
if llm_router is None or not isinstance(model_id, str):
return None
deployment: Final = _ROUTER_DEPLOYMENT.validate_python(
llm_router.get_model_info(id=model_id) # pyright: ignore[reportUnknownMemberType] # narrowed below
)
return deployment if is_str_keyed_mapping(deployment) else None
def wildcard_filled_model(model_name_pattern: str, litellm_model: str, listed_name: str) -> str | None:
listed_name_match: Final = re.match(PatternMatchRouter().pattern_to_regex(model_name_pattern), listed_name)
return (
None
if listed_name_match is None
else PatternMatchRouter.set_deployment_model_name(
matched_pattern=listed_name_match, litellm_deployment_litellm_model=litellm_model
)
)
def _deployment_routed_model(source: Mapping[str, object], listed_name: str) -> str | None:
source_pattern: Final = source.get("model_name")
source_model: Final = _litellm_model(source)
return (
wildcard_filled_model(source_pattern, source_model, listed_name)
if isinstance(source_pattern, str) and isinstance(source_model, str)
else None
)
def wildcard_routed_model(row: Mapping[str, object], llm_router: Router | None) -> str | None:
source: Final = _router_deployment(llm_router, _model_info_field(row, "id"))
listed_name: Final = row.get("model_name")
if llm_router is None or source is None or not isinstance(listed_name, str):
return None
if _litellm_model(row) == _litellm_model(source) or listed_name in llm_router.model_names:
return None
routed_model: Final = _deployment_routed_model(source, listed_name)
return (
routed_model
if routed_model is not None
and routed_model in llm_router.wildcard_routed_models(listed_name, _deployment_team_id(row))
else None
)
def _get_wildcard_models(
unique_models: list[str],
return_wildcard_routes: bool | None = False,

16
litellm/proxy/management_endpoints/model_management_endpoints.py
View file

@ -61,6 +61,7 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
)
from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY
from litellm.proxy.auth.model_checks import wildcard_filled_model
from litellm.proxy.auth.team_grants import team_model_aliases
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.config_sync_pubsub import (
@ -1264,9 +1265,18 @@ def _ptu_priced_deployment(model_params: Deployment) -> Deployment:
)
def _cost_map_entry(db_model: Deployment, incoming_model_info: Mapping[str, object]) -> Mapping[str, object]:
def _live_cost_map_model(db_model: Deployment, listed_name: str | None) -> str | None:
stored_model: Final = _decrypted_model(db_model.litellm_params.model)
if stored_model is None or listed_name is None:
return stored_model
return wildcard_filled_model(db_model.model_name, stored_model, listed_name) or stored_model
def _cost_map_entry(
db_model: Deployment, incoming_model_info: Mapping[str, object], listed_name: str | None
) -> Mapping[str, object]:
base_model: Final = incoming_model_info.get("base_model")
lookup: Final = base_model if isinstance(base_model, str) else _decrypted_model(db_model.litellm_params.model)
lookup: Final = base_model if isinstance(base_model, str) else _live_cost_map_model(db_model, listed_name)
if lookup is None:
return MappingProxyType({})
with suppress(Exception):
@ -1325,7 +1335,7 @@ def update_db_model(
incoming_model_info: Final = updated_patch.model_info.model_dump(exclude_none=True)
echoed_fields: Final = echoed_cost_map_fields(
incoming_model_info,
_cost_map_entry(db_model, incoming_model_info),
_cost_map_entry(db_model, incoming_model_info, updated_patch.model_name),
_loaded_catalog_entry(incoming_model_info, loaded_catalog),
)
stored_model_info_fields: Final[Mapping[str, object]] = without_server_derived_pricing(incoming_model_info)

13
litellm/proxy/proxy_server.py
View file

@ -441,6 +441,7 @@ from litellm.proxy.auth.model_checks import (
get_key_models,
get_mcp_server_ids,
get_team_models,
wildcard_routed_model,
)
from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy
from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # legacy module exports
@ -16083,12 +16084,18 @@ def _enrich_model_info_with_litellm_data(
# read litellm model_prices_and_context_window.json to get the following:
# input_cost_per_token, output_cost_per_token, max_tokens
litellm_model_info = get_litellm_model_info(model=model)
routed_model: Final = wildcard_routed_model(model, llm_router)
priced_model: Final = (
model
if routed_model is None
else {**model, "litellm_params": {**model["litellm_params"], "model": routed_model}}
)
litellm_model_info = get_litellm_model_info(model=priced_model)
# 2nd pass on the model, try seeing if we can find model in litellm model_cost map
if litellm_model_info == {}:
# use litellm_param model_name to get model_info
litellm_params = model.get("litellm_params", {})
litellm_params = priced_model.get("litellm_params", {})
litellm_model = litellm_params.get("model", None)
try:
litellm_model_info = litellm.get_model_info(model=litellm_model)
@ -16097,7 +16104,7 @@ def _enrich_model_info_with_litellm_data(
# 3rd pass on the model, try seeing if we can find model but without the "/" in model cost map
if litellm_model_info == {}:
# use litellm_param model_name to get model_info
litellm_params = model.get("litellm_params", {})
litellm_params = priced_model.get("litellm_params", {})
litellm_model = litellm_params.get("model", None)
if litellm_model:
split_model: Final = litellm_model.split("/")

22
tests/unit/proxy/management_endpoints/test_model_management_endpoints.py
View file

@ -4431,6 +4431,28 @@ class TestModelInfoCostMapEchoFilter:
"cost-map metadata must not be persisted from an unchanged /model/info echo"
)
def test_echo_of_an_expanded_wildcard_row_is_not_persisted_on_the_wildcard(self):
import litellm
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
entry = litellm.get_model_info("openai/gpt-5.6")
echo = {**entry, "id": "dep-echo-wildcard", "db_model": True, "access_groups": ["prod"]}
db_model = Deployment(
model_name="my-openai/*",
litellm_params=LiteLLM_Params(model="openai/*"),
model_info=ModelInfo(id="dep-echo-wildcard"),
)
result = update_db_model(
db_model=db_model,
updated_patch=updateDeployment(model_name="my-openai/gpt-5.6", model_info=ModelInfo(**echo)),
)
info = json.loads(result["model_info"])
assert info["access_groups"] == ["prod"]
assert set(info).isdisjoint(entry), "one listed model's metadata must not be saved onto the whole wildcard"
def test_an_edited_value_survives_the_echo_filter(self):
import litellm
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model

197
tests/unit/proxy/proxy_server/test_routes_model_info.py
View file

@ -578,6 +578,203 @@ async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monke
assert [model["model_name"] for model in result] == ["openai/gpt-4o"]
def test_model_info_prices_expanded_wildcard_rows_as_the_model_their_calls_route_to(
client: TestClient,
auth_as: Callable[[], AbstractContextManager[object]],
monkeypatch: pytest.MonkeyPatch,
local_model_cost_map: None,
) -> None:
"""A catch-all over Fireworks lists ``acme-chat``, which also names a DeepSeek entry, and a
prefixed OpenAI wildcard lists ``my-openai/gpt-acme``, which names no entry at all. Each row must
show the provider and price its calls are billed at."""
from litellm.proxy.auth import model_checks
catalog: Final = {
"fireworks_ai/accounts/fireworks/models/acme-chat": {
"input_cost_per_token": 1e-07,
"output_cost_per_token": 2e-07,
"litellm_provider": "fireworks_ai",
"mode": "chat",
},
"acme-chat": {
"input_cost_per_token": 9e-07,
"output_cost_per_token": 9e-06,
"litellm_provider": "deepseek",
"mode": "chat",
},
"gpt-acme": {
"input_cost_per_token": 3e-07,
"output_cost_per_token": 4e-07,
"litellm_provider": "openai",
"mode": "chat",
},
}
monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog})
_invalidate_model_cost_lowercase_map()
monkeypatch.setattr(
model_checks,
"get_provider_models",
lambda provider, litellm_params=None: {
"fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-chat"],
"openai": ["gpt-acme"],
}.get(provider, []),
)
router: Final = litellm.Router(
model_list=[
{
"model_name": "*",
"litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/*", "api_key": "x"},
"model_info": {"id": "fireworks-wildcard"},
},
{
"model_name": "my-openai/*",
"litellm_params": {"model": "openai/*", "api_key": "x"},
"model_info": {"id": "openai-wildcard"},
},
]
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list())
monkeypatch.setattr(proxy_server, "user_model", None)
with auth_as():
response: Final = client.get("/model/info")
assert response.status_code == 200, response.text
listed: Final = {
row["model_name"]: (
row["litellm_params"]["model"],
row["model_info"].get("litellm_provider"),
row["model_info"].get("input_cost_per_token"),
row["model_info"].get("output_cost_per_token"),
)
for row in response.json()["data"]
}
assert listed == {
"acme-chat": ("acme-chat", "fireworks_ai", 1e-07, 2e-07),
"my-openai/gpt-acme": ("my-openai/gpt-acme", "openai", 3e-07, 4e-07),
}
calls: Final = {
model_name: router.completion(
model=model_name, messages=[{"role": "user", "content": "hi"}], mock_response="hello there"
)
for model_name in listed
}
assert {model_name: call._hidden_params["response_cost"] for model_name, call in calls.items()} == pytest.approx(
{
model_name: listed[model_name][2] * call.usage.prompt_tokens
+ listed[model_name][3] * call.usage.completion_tokens
for model_name, call in calls.items()
}
)
_invalidate_model_cost_lowercase_map()
def test_model_info_never_prices_a_wildcard_row_as_a_route_its_calls_skip(
client: TestClient,
auth_as: Callable[[], AbstractContextManager[object]],
monkeypatch: pytest.MonkeyPatch,
local_model_cost_map: None,
) -> None:
"""``acme-lite`` has a deployment of its own and ``my-openai/gpt-acme-*`` is a narrower wildcard,
so calls to those names never reach the catch-all or ``my-openai/*``, which still list them. Such a
row may go unpriced, but any price a row shows must be the one its calls are billed at."""
from litellm.proxy.auth import model_checks
catalog: Final = {
"fireworks_ai/accounts/fireworks/models/acme-lite": {
"input_cost_per_token": 5e-07,
"output_cost_per_token": 6e-07,
"litellm_provider": "fireworks_ai",
"mode": "chat",
},
"deepseek/acme-lite": {
"input_cost_per_token": 7e-07,
"output_cost_per_token": 8e-07,
"litellm_provider": "deepseek",
"mode": "chat",
},
"gpt-acme-lite": {
"input_cost_per_token": 3e-07,
"output_cost_per_token": 4e-07,
"litellm_provider": "openai",
"mode": "chat",
},
}
monkeypatch.setattr(litellm, "model_cost", {**litellm.model_cost, **catalog})
_invalidate_model_cost_lowercase_map()
monkeypatch.setattr(
model_checks,
"get_provider_models",
lambda provider, litellm_params=None: {
"fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-lite"],
"openai": ["gpt-acme-lite"],
}.get(provider, []),
)
router: Final = litellm.Router(
model_list=[
{
"model_name": "*",
"litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/*", "api_key": "x"},
"model_info": {"id": "fireworks-wildcard"},
},
{
"model_name": "acme-lite",
"litellm_params": {"model": "deepseek/acme-lite", "api_key": "x"},
"model_info": {"id": "deepseek-acme-lite"},
},
{
"model_name": "my-openai/*",
"litellm_params": {"model": "openai/*", "api_key": "x"},
"model_info": {"id": "openai-wildcard"},
},
{
"model_name": "my-openai/gpt-acme-*",
"litellm_params": {"model": "deepseek/acme-*", "api_key": "x"},
"model_info": {"id": "deepseek-wildcard"},
},
]
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list())
monkeypatch.setattr(proxy_server, "user_model", None)
with auth_as():
response: Final = client.get("/model/info")
assert response.status_code == 200, response.text
listed: Final = {
(row["model_info"]["id"], row["model_name"]): (
row["model_info"].get("input_cost_per_token"),
row["model_info"].get("output_cost_per_token"),
)
for row in response.json()["data"]
if "*" not in row["model_name"]
}
assert sorted(listed) == [
("deepseek-acme-lite", "acme-lite"),
("fireworks-wildcard", "acme-lite"),
("openai-wildcard", "my-openai/gpt-acme-lite"),
]
assert listed[("deepseek-acme-lite", "acme-lite")] == (7e-07, 8e-07)
calls: Final = {
model_name: router.completion(
model=model_name, messages=[{"role": "user", "content": "hi"}], mock_response="hello there"
)
for _, model_name in listed
}
priced_rows: Final = {key: prices for key, prices in listed.items() if prices[0] is not None}
assert {
key: calls[key[1]]._hidden_params["response_cost"] for key in priced_rows
} == pytest.approx(
{
key: input_cost * calls[key[1]].usage.prompt_tokens + output_cost * calls[key[1]].usage.completion_tokens
for key, (input_cost, output_cost) in priced_rows.items()
}
)
_invalidate_model_cost_lowercase_map()
# ---------------------------------------------------------------------------
# GET /model/info — team BYOK scoping (issue #30983)
# ---------------------------------------------------------------------------