mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
fix(proxy): price a wildcard row aliased to a sibling name as the alias target
This commit is contained in:
parent
fa4b26fc9f
commit
2083957b33
3 changed files with 61 additions and 23 deletions
|
|
@ -582,25 +582,29 @@ def wildcard_row_priced_model(row: Mapping[str, object], llm_router: Router | No
|
|||
return wildcard_listed_priced_model(
|
||||
source_pattern,
|
||||
source_model,
|
||||
listed_name,
|
||||
routed_model_name(llm_router, listed_name),
|
||||
routes_through_wildcard=routes_through_deployment(llm_router, model_id, listed_name, _deployment_team_id(row)),
|
||||
)
|
||||
|
||||
|
||||
def wildcard_listed_priced_model(
|
||||
model_name_pattern: str, wildcard_model: str, listed_name: str, routes_through_wildcard: bool
|
||||
) -> str | None:
|
||||
model_name_pattern: str, wildcard_model: str, routed_name: str, routes_through_wildcard: bool
|
||||
) -> str:
|
||||
if not routes_through_wildcard:
|
||||
return wildcard_model
|
||||
filled: Final = wildcard_filled_model(model_name_pattern, wildcard_model, listed_name)
|
||||
filled: Final = wildcard_filled_model(model_name_pattern, wildcard_model, routed_name)
|
||||
if filled is None:
|
||||
return wildcard_model
|
||||
return None if _repeats_listed_provider_prefix(filled, listed_name) else filled
|
||||
return routed_name if _repeats_provider_prefix(filled, routed_name) else filled
|
||||
|
||||
|
||||
def _repeats_listed_provider_prefix(filled_model: str, listed_name: str) -> bool:
|
||||
listed_provider, separator, _ = listed_name.partition("/")
|
||||
return separator != "" and filled_model == f"{listed_provider}/{listed_name}"
|
||||
def _repeats_provider_prefix(filled_model: str, routed_name: str) -> bool:
|
||||
provider, separator, _ = routed_name.partition("/")
|
||||
return separator != "" and filled_model == f"{provider}/{routed_name}"
|
||||
|
||||
|
||||
def routed_model_name(llm_router: Router, listed_name: str) -> str:
|
||||
return llm_router.registry().resolve(listed_name, kind="alias") or listed_name
|
||||
|
||||
|
||||
def routes_through_deployment(llm_router: Router, model_id: str, listed_name: str, team_id: str | None) -> bool:
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ from litellm.proxy._types import (
|
|||
from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY
|
||||
from litellm.proxy.auth.model_checks import (
|
||||
model_info_cost_map_entry,
|
||||
routed_model_name,
|
||||
routes_through_deployment,
|
||||
wildcard_listed_priced_model,
|
||||
)
|
||||
|
|
@ -1275,15 +1276,14 @@ def _live_cost_map_model(db_model: Deployment, listed_name: str | None, llm_rout
|
|||
if stored_model is None or listed_name is None:
|
||||
return stored_model
|
||||
model_id: Final = db_model.model_info.id
|
||||
priced_as: Final = wildcard_listed_priced_model(
|
||||
return wildcard_listed_priced_model(
|
||||
db_model.model_name,
|
||||
stored_model,
|
||||
listed_name,
|
||||
listed_name if llm_router is None else routed_model_name(llm_router, listed_name),
|
||||
routes_through_wildcard=llm_router is None
|
||||
or model_id is None
|
||||
or routes_through_deployment(llm_router, model_id, listed_name, db_model.model_info.team_id),
|
||||
)
|
||||
return listed_name if priced_as is None else priced_as
|
||||
|
||||
|
||||
def _cost_map_entry(
|
||||
|
|
|
|||
|
|
@ -904,6 +904,18 @@ def _catch_all_and_shadowed_wildcard_router(monkeypatch: pytest.MonkeyPatch) ->
|
|||
"litellm_provider": "fireworks_ai",
|
||||
"mode": "chat",
|
||||
},
|
||||
"fireworks_ai/accounts/fireworks/models/acme-max": {
|
||||
"input_cost_per_token": 9e-07,
|
||||
"output_cost_per_token": 9.5e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"mode": "chat",
|
||||
},
|
||||
"fireworks_ai/accounts/fireworks/models/acme-mini": {
|
||||
"input_cost_per_token": 5e-08,
|
||||
"output_cost_per_token": 6e-08,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"mode": "chat",
|
||||
},
|
||||
"deepseek/acme-lite": {
|
||||
"input_cost_per_token": 7e-07,
|
||||
"output_cost_per_token": 8e-07,
|
||||
|
|
@ -918,7 +930,11 @@ def _catch_all_and_shadowed_wildcard_router(monkeypatch: pytest.MonkeyPatch) ->
|
|||
"get_provider_models",
|
||||
lambda provider, litellm_params=None: {
|
||||
"openai": ["gpt-5.6-acme"],
|
||||
"fireworks_ai": ["fireworks_ai/accounts/fireworks/models/acme-pro"],
|
||||
"fireworks_ai": [
|
||||
"fireworks_ai/accounts/fireworks/models/acme-pro",
|
||||
"fireworks_ai/accounts/fireworks/models/acme-max",
|
||||
"fireworks_ai/accounts/fireworks/models/acme-mini",
|
||||
],
|
||||
}.get(provider, []),
|
||||
)
|
||||
router: Final = litellm.Router(
|
||||
|
|
@ -939,7 +955,7 @@ def _catch_all_and_shadowed_wildcard_router(monkeypatch: pytest.MonkeyPatch) ->
|
|||
"model_info": {"id": "deepseek-acme-lite"},
|
||||
},
|
||||
],
|
||||
model_group_alias={"my-fw/acme-pro": "acme-lite"},
|
||||
model_group_alias={"my-fw/acme-pro": "acme-lite", "my-fw/acme-max": "my-fw/acme-mini"},
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list())
|
||||
|
|
@ -983,6 +999,30 @@ def test_model_info_prices_a_catch_all_row_named_with_its_provider_as_the_listed
|
|||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_model_info_prices_a_wildcard_row_aliased_to_a_sibling_name_as_the_alias_target(
|
||||
client: TestClient,
|
||||
auth_as: Callable[[], AbstractContextManager[object]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
local_model_cost_map: None,
|
||||
) -> None:
|
||||
"""An alias from ``my-fw/acme-max`` to ``my-fw/acme-mini`` still reaches the same wildcard, but calls run
|
||||
``acme-mini``, so the ``my-fw/acme-max`` row must show the price those calls are billed at."""
|
||||
router: Final = _catch_all_and_shadowed_wildcard_router(monkeypatch)
|
||||
|
||||
with auth_as():
|
||||
rows: Final = _expanded_wildcard_rows(client)
|
||||
|
||||
shown: Final = rows["my-fw/acme-max"]["model_info"]
|
||||
call: Final = router.completion(
|
||||
model="my-fw/acme-max", messages=[{"role": "user", "content": "hi"}], mock_response="hello there"
|
||||
)
|
||||
assert call._hidden_params["response_cost"] == pytest.approx(
|
||||
shown["input_cost_per_token"] * call.usage.prompt_tokens
|
||||
+ shown["output_cost_per_token"] * call.usage.completion_tokens
|
||||
)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_saving_a_replayed_model_info_wildcard_row_stores_none_of_its_shown_metadata(
|
||||
client: TestClient,
|
||||
auth_as: Callable[[], AbstractContextManager[object]],
|
||||
|
|
@ -991,7 +1031,7 @@ def test_saving_a_replayed_model_info_wildcard_row_stores_none_of_its_shown_meta
|
|||
) -> None:
|
||||
"""An API client that saves a ``/model/info`` row back unchanged, apart from one edit, must not pin the
|
||||
cost-map metadata the row showed onto the wildcard, whether the row is a catch-all name carrying its
|
||||
provider or a name an alias sends to another deployment."""
|
||||
provider, a name an alias sends to another deployment, or a name an alias sends to a sibling name."""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment
|
||||
|
||||
|
|
@ -999,7 +1039,7 @@ def test_saving_a_replayed_model_info_wildcard_row_stores_none_of_its_shown_meta
|
|||
with auth_as():
|
||||
rows: Final = _expanded_wildcard_rows(client)
|
||||
|
||||
assert sorted(rows) == ["my-fw/acme-pro", "openai/gpt-5.6-acme"]
|
||||
assert sorted(rows) == ["my-fw/acme-max", "my-fw/acme-mini", "my-fw/acme-pro", "openai/gpt-5.6-acme"]
|
||||
sources: Final = {
|
||||
deployment["model_info"]["id"]: deployment for deployment in router.get_model_list() or []
|
||||
}
|
||||
|
|
@ -1023,14 +1063,8 @@ def test_saving_a_replayed_model_info_wildcard_row_stores_none_of_its_shown_meta
|
|||
name: set(row["model_info"]) - set(sources[row["model_info"]["id"]]["model_info"]) for name, row in rows.items()
|
||||
}
|
||||
assert all(shown_metadata.values()), shown_metadata
|
||||
assert {name: info["access_groups"] for name, info in stored.items()} == {
|
||||
"my-fw/acme-pro": ["prod"],
|
||||
"openai/gpt-5.6-acme": ["prod"],
|
||||
}
|
||||
assert {name: shown_metadata[name] & set(info) for name, info in stored.items()} == {
|
||||
"my-fw/acme-pro": set(),
|
||||
"openai/gpt-5.6-acme": set(),
|
||||
}
|
||||
assert {name: info["access_groups"] for name, info in stored.items()} == dict.fromkeys(rows, ["prod"])
|
||||
assert {name: shown_metadata[name] & set(info) for name, info in stored.items()} == dict.fromkeys(rows, set())
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue