scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 156

Merge e591c18ce6 into 9071ca503e

Browse source
This commit is contained in:
Deepanshu Lulla 2026-09-12 08:25:17 -04:00 • committed by GitHub
commit 71b46785eb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 238 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
litellm/router_strategy/tag_based_routing.py
View file

@ -434,6 +434,36 @@ def _inherited_constraint_sets(
return frozenset(inherited_required), frozenset(inherited_excluded)
def _inherited_full_tags(inherited_tags: object, routing_prefix: str) -> frozenset[str]:
# Policy tags exempt request-tag scoping regardless of prefix (see
# _scope_to_confirmed_and_inherited_tags); matched by exact rewritten form,
# not bare value, so a caller can't exempt an unrelated tag by reusing a
# policy tag's bare value under a different "&"/"!" marker.
if not isinstance(inherited_tags, (list, tuple)):
return frozenset()
return frozenset(_strip_routing_prefix(inherited_tags, routing_prefix)[0])
def _scope_to_confirmed_and_inherited_tags(
rewritten_tags: tuple[str, ...],
original_tags: Sequence[str],
inherited_tags: object,
routing_prefix: str,
) -> tuple[str, ...]:
# Once a prefix is configured, an unprefixed request tag is noise, not a
# routing signal -- except a tag the caller explicitly prefixed, or one
# policy already contributed via inherited_tags (not caller-controlled, so
# exempt regardless of prefix). Matching the full rewritten tag rather than
# _strip_routing_prefix's bare-value `confirmed` return (used elsewhere for
# the "known required tag" fail-open check) avoids exempting an unrelated
# tag that happens to share a bare value under a different marker.
confirmed_full_tags: Final = frozenset(
rewritten for rewritten, original in zip(rewritten_tags, original_tags) if original.startswith(routing_prefix)
)
inherited_full_tags: Final = _inherited_full_tags(inherited_tags, routing_prefix)
return tuple(t for t in rewritten_tags if t in confirmed_full_tags or t in inherited_full_tags)
def _tag_known_to_group(
llm_router_instance: LitellmRouter,
model: str,
@ -532,7 +562,14 @@ async def get_deployments_for_tag(
# caller-declared routing directive, exempt from the "maybe foreign to this
# group" heuristics that unprefixed tags still go through unchanged below.
rewritten_tags, routing_confirmed = _strip_routing_prefix(request_tags or [], routing_prefix)
required_tags, positive_tags, excluded_patterns = _split_tags(rewritten_tags)
scoped_tags: Final = (
_scope_to_confirmed_and_inherited_tags(
rewritten_tags, request_tags or (), metadata.get("inherited_tags"), routing_prefix
)
if routing_prefix
else rewritten_tags
)
required_tags, positive_tags, excluded_patterns = _split_tags(scoped_tags)
inherited_required_set, inherited_excluded_set = _inherited_constraint_sets(
metadata.get("inherited_tags"), routing_prefix
)

200
tests/test_litellm/router_strategy/test_router_tag_routing.py
View file

@ -3160,3 +3160,203 @@ async def test_chat_request_carrying_litellm_metadata_still_routes_on_proxy_merg
)
assert deployment["model_info"]["id"] == "team-a-deployment"
# --- unprefixed request tags must be a no-op once tag_routing_prefix is
# configured (GitHub issue #39901) ---
def _untagged_single_deployment_router(tag_routing_prefix: str = "route:") -> litellm.Router:
return litellm.Router(
model_list=[
{
"model_name": "chat",
"litellm_params": {
"model": "gpt-4o",
"api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
},
"model_info": {"id": "only-deployment"},
},
],
enable_tag_filtering=True,
tag_routing_prefix=tag_routing_prefix,
)
@pytest.mark.asyncio()
async def test_unprefixed_request_tag_is_noop_when_prefix_configured():
# Zero deployment tags in the group; an unrelated tag must not raise.
router = _untagged_single_deployment_router()
response = await router.acompletion(
model="chat",
messages=[{"role": "user", "content": "hi"}],
metadata={"tags": ["user_id:234"]},
mock_response="hi",
)
assert response._hidden_params["model_id"] == "only-deployment"
@pytest.mark.asyncio()
async def test_no_tags_still_noop_when_prefix_configured():
# Baseline: no tags at all must keep working identically.
router = _untagged_single_deployment_router()
response = await router.acompletion(
model="chat",
messages=[{"role": "user", "content": "hi"}],
metadata={"tags": []},
mock_response="hi",
)
assert response._hidden_params["model_id"] == "only-deployment"
@pytest.mark.asyncio()
async def test_prefixed_tag_still_raises_when_unresolvable():
# A genuinely prefixed, unresolvable tag must still raise.
router = _untagged_single_deployment_router()
with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info:
await router.acompletion(
model="chat",
messages=[{"role": "user", "content": "hi"}],
metadata={"tags": ["route:quality:nonexistent"]},
mock_response="hi",
)
from litellm.types.router import RouterErrors
assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value)
@pytest.mark.asyncio()
async def test_unprefixed_tag_still_raises_when_no_prefix_configured():
# Backward compat: no prefix configured, so the old behavior is unchanged.
router = _untagged_single_deployment_router(tag_routing_prefix="")
with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info:
await router.acompletion(
model="chat",
messages=[{"role": "user", "content": "hi"}],
metadata={"tags": ["user_id:234"]},
mock_response="hi",
)
from litellm.types.router import RouterErrors
assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value)
@pytest.mark.asyncio()
async def test_inherited_required_tag_unaffected_by_prefix_scoping():
# An unprefixed policy "&" requirement in inherited_tags must still apply.
router = _eu_region_router()
router.update_settings(tag_routing_prefix="route:")
response = await router.acompletion(
model="chat",
messages=[{"role": "user", "content": "hi"}],
metadata={"tags": ["&region:eu"], "inherited_tags": ["&region:eu"]},
mock_response="hi",
)
assert response._hidden_params["model_id"] == "eu-1"
@pytest.mark.asyncio()
async def test_inherited_excluded_tag_unaffected_by_prefix_scoping():
# Same, for a "!" exclusion, without exhausting the pool -- proves the
# primary candidate computation honors it, not just the fail-open floor.
router = _eu_region_router()
router.update_settings(tag_routing_prefix="route:")
response = await router.acompletion(
model="chat",
messages=[{"role": "user", "content": "hi"}],
metadata={"tags": ["!region:us"], "inherited_tags": ["!region:us"]},
mock_response="hi",
)
assert response._hidden_params["model_id"] == "eu-1"
@pytest.mark.asyncio()
async def test_foreign_tag_ignored_alongside_inherited_constraint_when_prefix_configured():
# A foreign tag and an inherited requirement in the same request: the
# foreign tag must be ignored. No allow_fail_open (unlike _eu_region_router)
# so a leaked foreign tag can't self-heal via the fail-open floor.
router = litellm.Router(
model_list=[
{
"model_name": "chat",
"litellm_params": {
"model": "gpt-4o",
"api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
"tags": ["region:eu"],
},
"model_info": {"id": "eu-1"},
},
{
"model_name": "chat",
"litellm_params": {
"model": "gpt-4o-mini",
"api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
"tags": ["region:us"],
},
"model_info": {"id": "us-1"},
},
],
enable_tag_filtering=True,
)
router.update_settings(tag_routing_prefix="route:")
response = await router.acompletion(
model="chat",
messages=[{"role": "user", "content": "hi"}],
metadata={"tags": ["&region:eu", "user_id:234"], "inherited_tags": ["&region:eu"]},
mock_response="hi",
)
assert response._hidden_params["model_id"] == "eu-1"
@pytest.mark.asyncio()
async def test_unprefixed_tag_cannot_piggyback_via_bare_value_collision_with_confirmed_tag():
# Exemption is keyed on the exact tag, not its bare value: a confirmed
# "route:region:eu" must not exempt an unrelated "!region:eu" sharing the
# same bare value under a different marker.
router = litellm.Router(
model_list=[
{
"model_name": "chat",
"litellm_params": {
"model": "gpt-4o",
"api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
"tags": ["region:eu"],
},
"model_info": {"id": "eu-1"},
},
{
"model_name": "chat",
"litellm_params": {
"model": "gpt-4o-mini",
"api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
"tags": ["region:us"],
},
"model_info": {"id": "us-1"},
},
],
enable_tag_filtering=True,
tag_routing_prefix="route:",
)
response = await router.acompletion(
model="chat",
messages=[{"role": "user", "content": "hi"}],
metadata={"tags": ["route:region:eu", "!region:eu"]},
mock_response="hi",
)
assert response._hidden_params["model_id"] == "eu-1"