mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): count routing-group and alias members with the caller's team, as routing serves them
This commit is contained in:
parent
1cc994bfd4
commit
89c34f9d34
2 changed files with 32 additions and 6 deletions
|
|
@ -146,7 +146,8 @@ def _count_input_tokens_for_models(
|
|||
|
||||
def models_counted_for(model: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]:
|
||||
"""The names to count a request for `model` from `team_id` with, keeping the largest count, one per deployment
|
||||
routing would serve it from: the ones named `model`, else those of the one wildcard route that serves it.
|
||||
routing would serve it from: the team's or else the shared ones named `model`, its alias target, or its routing
|
||||
group's members, else those of the one wildcard route that serves it.
|
||||
|
||||
An OpenAI deployment counts as its OpenAI model, so it gets that model's tokenizer and PDF page pricing, prefixed
|
||||
`openai/` when OpenAI itself serves it, so `openai_chat_pdf_file_tokens` can add what its chat completions bills. A
|
||||
|
|
@ -160,12 +161,13 @@ def models_counted_for(model: str, llm_router: Router | None, team_id: str | Non
|
|||
|
||||
def _routed_deployments(model: str, llm_router: Router, team_id: str | None) -> Sequence[object]:
|
||||
registry: Final = llm_router.registry()
|
||||
named: Final = (
|
||||
*registry.deployments(model, source="all", team_id=team_id),
|
||||
*registry.deployments(model, source="alias"),
|
||||
*registry.deployments(model, source="routing_groups"),
|
||||
resolved: Final = registry.resolve(model, kind="alias") or model
|
||||
routing_group: Final = llm_router.get_routing_group(resolved)
|
||||
members: Final = routing_group.models if routing_group is not None else (resolved,)
|
||||
named: Final = tuple(
|
||||
chain.from_iterable(registry.deployments(member, source="all", team_id=team_id) for member in members)
|
||||
)
|
||||
return named or registry.deployments(model, source="wildcard", team_id=team_id)
|
||||
return named or registry.deployments(resolved, source="wildcard", team_id=team_id)
|
||||
|
||||
|
||||
def openai_chat_pdf_file_tokens(counted_model: str, pdf_files: int) -> int:
|
||||
|
|
|
|||
|
|
@ -159,6 +159,30 @@ def test_a_team_request_counts_with_the_deployment_its_team_is_served_by():
|
|||
assert models_counted_for("reports", router, "team-b") == ("reports",)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("requested", ["reports", "latest-reports"], ids=["routing-group", "alias-of-routing-group"])
|
||||
def test_a_team_request_to_a_routing_group_counts_with_the_member_deployment_its_team_is_served_by(requested: str):
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "drafts",
|
||||
"litellm_params": {"model": "azure/gpt-5.6-sol", "api_key": "x", "api_base": "https://x"},
|
||||
},
|
||||
{
|
||||
"model_name": "drafts-team-a",
|
||||
"litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"},
|
||||
"model_info": {"team_id": "team-a", "team_public_model_name": "drafts"},
|
||||
},
|
||||
],
|
||||
routing_groups=[{"group_name": "reports", "models": ["drafts"], "routing_strategy": "simple-shuffle"}],
|
||||
model_group_alias={"latest-reports": "reports"},
|
||||
)
|
||||
|
||||
assert models_counted_for(requested, router, "team-a") == models_counted_for(
|
||||
requested, _router((requested, "anthropic/claude-haiku-5-5")), None
|
||||
)
|
||||
assert models_counted_for(requested, router, "team-b") == ("azure/gpt-5.6-sol",)
|
||||
|
||||
|
||||
def test_a_team_with_its_own_wildcard_counts_with_it_and_never_with_a_proxy_wide_wildcard():
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue