fix(proxy): count routing-group and alias members with the caller's team, as routing serves them

This commit is contained in:
mateo-berri 2026-10-29 15:10:29 -07:00
parent 1cc994bfd4
commit 89c34f9d34
2 changed files with 32 additions and 6 deletions

View file

@ -146,7 +146,8 @@ def _count_input_tokens_for_models(
def models_counted_for(model: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]:
"""The names to count a request for `model` from `team_id` with, keeping the largest count, one per deployment
routing would serve it from: the ones named `model`, else those of the one wildcard route that serves it.
routing would serve it from: the team's or else the shared ones named `model`, its alias target, or its routing
group's members, else those of the one wildcard route that serves it.
An OpenAI deployment counts as its OpenAI model, so it gets that model's tokenizer and PDF page pricing, prefixed
`openai/` when OpenAI itself serves it, so `openai_chat_pdf_file_tokens` can add what its chat completions bills. A
@ -160,12 +161,13 @@ def models_counted_for(model: str, llm_router: Router | None, team_id: str | Non
def _routed_deployments(model: str, llm_router: Router, team_id: str | None) -> Sequence[object]:
registry: Final = llm_router.registry()
named: Final = (
*registry.deployments(model, source="all", team_id=team_id),
*registry.deployments(model, source="alias"),
*registry.deployments(model, source="routing_groups"),
resolved: Final = registry.resolve(model, kind="alias") or model
routing_group: Final = llm_router.get_routing_group(resolved)
members: Final = routing_group.models if routing_group is not None else (resolved,)
named: Final = tuple(
chain.from_iterable(registry.deployments(member, source="all", team_id=team_id) for member in members)
)
return named or registry.deployments(model, source="wildcard", team_id=team_id)
return named or registry.deployments(resolved, source="wildcard", team_id=team_id)
def openai_chat_pdf_file_tokens(counted_model: str, pdf_files: int) -> int:

View file

@ -159,6 +159,30 @@ def test_a_team_request_counts_with_the_deployment_its_team_is_served_by():
assert models_counted_for("reports", router, "team-b") == ("reports",)
@pytest.mark.parametrize("requested", ["reports", "latest-reports"], ids=["routing-group", "alias-of-routing-group"])
def test_a_team_request_to_a_routing_group_counts_with_the_member_deployment_its_team_is_served_by(requested: str):
router: Final = Router(
model_list=[
{
"model_name": "drafts",
"litellm_params": {"model": "azure/gpt-5.6-sol", "api_key": "x", "api_base": "https://x"},
},
{
"model_name": "drafts-team-a",
"litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"},
"model_info": {"team_id": "team-a", "team_public_model_name": "drafts"},
},
],
routing_groups=[{"group_name": "reports", "models": ["drafts"], "routing_strategy": "simple-shuffle"}],
model_group_alias={"latest-reports": "reports"},
)
assert models_counted_for(requested, router, "team-a") == models_counted_for(
requested, _router((requested, "anthropic/claude-haiku-5-5")), None
)
assert models_counted_for(requested, router, "team-b") == ("azure/gpt-5.6-sol",)
def test_a_team_with_its_own_wildcard_counts_with_it_and_never_with_a_proxy_wide_wildcard():
router: Final = Router(
model_list=[