scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge pull request #36628 from BerriAI/litellm_fix_autorouter_consumed_tags

Browse source
This commit is contained in:
Mateo Wang 2026-08-12 11:35:15 -07:00 • committed by GitHub
commit f082f18e2e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 443 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
basedpyright-code-budget.json
View file

@ -1,6 +1,6 @@
{
"reportAny": {
"limit": 23919
"limit": 23914
},
"reportArgumentType": {
"limit": 2580

1
litellm/constants.py
View file

@ -1325,6 +1325,7 @@ LITELLM_METADATA_FIELD: Final = "litellm_metadata"
OLD_LITELLM_METADATA_FIELD: Final = "metadata"
RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name"
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl"
CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated"
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = (

7
litellm/proxy/common_utils/callback_utils.py
View file

@ -6,7 +6,11 @@ from typing import TYPE_CHECKING, Any, Final, Optional
import litellm
from litellm import get_secret
from litellm._logging import verbose_proxy_logger
from litellm.constants import PRE_CALL_EXECUTED_GUARDRAILS_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY
from litellm.constants import (
CONSUMED_REQUEST_TAGS_METADATA_KEY,
PRE_CALL_EXECUTED_GUARDRAILS_KEY,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
@ -426,6 +430,7 @@ LITELLM_PROXY_INTERNAL_METADATA_KEYS: Final = frozenset(
"_pipeline_managed_guardrails",
PRE_CALL_EXECUTED_GUARDRAILS_KEY,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
CONSUMED_REQUEST_TAGS_METADATA_KEY,
"disable_global_guardrails",
"disable_global_guardrail",
"opted_out_global_guardrails",

2
litellm/proxy/litellm_pre_call_utils.py
View file

@ -16,6 +16,7 @@ import litellm
from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm._service_logger import ServiceLogging
from litellm.constants import (
CONSUMED_REQUEST_TAGS_METADATA_KEY,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
LITELLM_PROXY_MASTER_KEY_ALIAS,
PRE_CALL_EXECUTED_GUARDRAILS_KEY,
@ -261,6 +262,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = (
"policy_sources",
"routing_decision",
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
CONSUMED_REQUEST_TAGS_METADATA_KEY,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
"standard_logging_object",
"proxy_server_request",

56
litellm/router.py
View file

@ -43,6 +43,7 @@ from litellm.caching.caching import (
RedisClusterCache,
)
from litellm.constants import (
CONSUMED_REQUEST_TAGS_METADATA_KEY,
DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS,
DEFAULT_HEALTH_CHECK_INTERVAL,
DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER,
@ -171,6 +172,7 @@ from litellm.types.router import (
AlertingConfig,
AllowedFailsPolicy,
AssistantsTypedDict,
ConsumedRequestTagsStamp,
CredentialLiteLLMParams,
CustomRoutingStrategyBase,
Deployment,
@ -11339,11 +11341,15 @@ class Router:
return filtered
def _select_pre_routing_strategy(self, model: str, request_kwargs: dict) -> "PreRoutingStrategy | None":
def _select_pre_routing_strategy(
self, model: str, request_kwargs: dict
) -> "TaggedPreRoutingStrategy[PreRoutingStrategy] | None":
"""
Resolve the pre-routing strategy for `model`, disambiguating deployments
that share a `model_name` by matching the request's tags against each
registered strategy's tags before falling back to the first registered.
Returns the tagged registry entry so the caller can tell whether the
request's tags were what selected it.
"""
candidates: Final[list[TaggedPreRoutingStrategy[PreRoutingStrategy]]] = [
*self.auto_routers.get(model, []),
@ -11354,7 +11360,7 @@ class Router:
if not candidates:
return None
if len(candidates) == 1:
return candidates[0].strategy
return candidates[0]
request_tags: Final = _get_tags_from_request_kwargs(request_kwargs)
if request_tags:
@ -11362,11 +11368,11 @@ class Router:
if tagged.tags and is_valid_deployment_tag(
list(tagged.tags), request_tags, self.tag_filtering_match_any
):
return tagged.strategy
return tagged
for tagged in candidates:
if "default" in tagged.tags:
return tagged.strategy
return candidates[0].strategy
return tagged
return candidates[0]
async def async_pre_routing_hook(
self,
@ -11390,15 +11396,18 @@ class Router:
if self.routing_plugins:
await self._run_routing_plugins(model=model, request_kwargs=request_kwargs, messages=messages)
router_strategy: Final = self._select_pre_routing_strategy(model=model, request_kwargs=request_kwargs)
if router_strategy is None:
selected_strategy: Final = self._select_pre_routing_strategy(model=model, request_kwargs=request_kwargs)
if selected_strategy is None:
self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
self._stamp_or_clear_metadata_key(
request_kwargs=request_kwargs, key=SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, value=None
)
self._stamp_or_clear_metadata_key(
request_kwargs=request_kwargs, key=CONSUMED_REQUEST_TAGS_METADATA_KEY, value=None
)
return None
pre_routing_hook_response: Final = await router_strategy.async_pre_routing_hook(
pre_routing_hook_response: Final = await selected_strategy.strategy.async_pre_routing_hook(
model=model,
request_kwargs=request_kwargs,
messages=messages,
@ -11414,6 +11423,15 @@ class Router:
key=SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
value=(pre_routing_hook_response.session_affinity_ttl_seconds if pre_routing_hook_response else None),
)
self._stamp_or_clear_metadata_key(
request_kwargs=request_kwargs,
key=CONSUMED_REQUEST_TAGS_METADATA_KEY,
value=self._consumed_request_tags_stamp(
selected_strategy=selected_strategy,
pre_routing_hook_response=pre_routing_hook_response,
request_tags=_get_tags_from_request_kwargs(request_kwargs),
),
)
# `model` (the alias, e.g. "smart-router") is never the deployment actually
# called - apply the alias's own litellm_params (besides `model` itself,
@ -11432,6 +11450,28 @@ class Router:
return pre_routing_hook_response
def _consumed_request_tags_stamp(
self,
selected_strategy: "TaggedPreRoutingStrategy[PreRoutingStrategy]",
pre_routing_hook_response: PreRoutingHookResponse | None,
request_tags: Sequence[str],
) -> ConsumedRequestTagsStamp | None:
"""Record which tags picked the router and which model group it rewrote to, or None.
A request whose tags matched the selected strategy's tags has already spent those
tags on picking the router; re-applying them to the routed tier's model group would
empty the pool unless every tier deployment repeats the marker's tag. Only the
strategy's own tags are spent: the request's other tags keep constraining
deployment selection inside the routed group, and key/team policy tags are
untouched because tag filtering separately re-applies whatever
`metadata.inherited_tags` carries for the stamped group.
"""
if pre_routing_hook_response is None or not selected_strategy.tags or not request_tags:
return None
if not is_valid_deployment_tag(selected_strategy.tags, request_tags, self.tag_filtering_match_any):
return None
return ConsumedRequestTagsStamp(model_group=pre_routing_hook_response.model, tags=selected_strategy.tags)
@staticmethod
def _record_routing_decision(
request_kwargs: dict,

41
litellm/router_strategy/tag_based_routing.py
View file

@ -13,7 +13,9 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal
from litellm._logging import verbose_logger
from litellm.types.router import RouterErrors
from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
from litellm.types.router import ConsumedRequestTagsStamp, RouterErrors
if TYPE_CHECKING:
from litellm.router import Router as _Router
@ -46,7 +48,9 @@ def _is_valid_deployment_tag_regex(
return None
def is_valid_deployment_tag(deployment_tags: list[str], request_tags: list[str], match_any: bool = True) -> bool:
def is_valid_deployment_tag(
deployment_tags: Sequence[str], request_tags: Sequence[str], match_any: bool = True
) -> bool:
"""
Check if a tag is valid, the matching can be either any or all based on `match_any` flag
"""
@ -389,6 +393,25 @@ def _tag_known_to_group(
)
def _request_tags_after_router_consumption(metadata: Mapping[Any, Any], model: str) -> Sequence[str] | None:
# The pre-routing hook stamps which tags selected the router it rewrote the request
# to: those tags already did their job and must not also constrain deployment choice
# inside the routed group. The request's other tags still apply there, on top of the
# inherited_tags snapshot that keeps key/team policy applying. Every other model
# group keeps the full list.
stamp: Final = metadata.get(CONSUMED_REQUEST_TAGS_METADATA_KEY)
if not isinstance(stamp, ConsumedRequestTagsStamp) or stamp.model_group != model:
return metadata.get("tags")
request_tags: Final = metadata.get("tags")
leftover: Final = tuple(
tag for tag in (request_tags if isinstance(request_tags, (list, tuple)) else ()) if tag not in stamp.tags
)
inherited_tags: Final = metadata.get("inherited_tags")
if not isinstance(inherited_tags, (list, tuple)):
return leftover or None
return tuple(dict.fromkeys((*leftover, *inherited_tags)))
async def get_deployments_for_tag(
llm_router_instance: LitellmRouter,
model: str, # used to raise the correct error
@ -429,7 +452,7 @@ async def get_deployments_for_tag(
verbose_logger.debug("request metadata: %s", request_kwargs.get(metadata_variable_name))
if metadata_variable_name in request_kwargs:
metadata: Final = request_kwargs[metadata_variable_name]
request_tags: Final = metadata.get("tags")
request_tags: Final = _request_tags_after_router_consumption(metadata, model)
match_any: Final = llm_router_instance.tag_filtering_match_any
routing_prefix: Final = llm_router_instance.tag_routing_prefix or ""
@ -563,26 +586,30 @@ async def get_deployments_for_tag(
def _get_tags_from_request_kwargs(
request_kwargs: dict[Any, Any] | None = None,
metadata_variable_name: Literal["metadata", "litellm_metadata"] = "metadata",
metadata_variable_name: Literal["metadata", "litellm_metadata"] | None = None,
) -> list[str]:
"""
Helper to get tags from request kwargs
Args:
request_kwargs: The request kwargs to get tags from
metadata_variable_name: Which metadata dict holds proxy metadata; resolved
from the kwargs when not pinned, so /v1/messages-shaped requests
(``litellm_metadata``) read the same bucket the proxy wrote tags to
Returns:
List[str]: The tags from the request kwargs
"""
if request_kwargs is None:
return []
if metadata_variable_name in request_kwargs:
metadata: Final = request_kwargs[metadata_variable_name] or {}
resolved_variable_name: Final = metadata_variable_name or get_metadata_variable_name_from_kwargs(request_kwargs)
if resolved_variable_name in request_kwargs:
metadata: Final = request_kwargs[resolved_variable_name] or {}
tags = metadata.get("tags", [])
return tags if tags is not None else []
elif "litellm_params" in request_kwargs:
litellm_params: Final = request_kwargs["litellm_params"] or {}
_metadata: Final = litellm_params.get(metadata_variable_name, {}) or {}
_metadata: Final = litellm_params.get(resolved_variable_name, {}) or {}
tags = _metadata.get("tags", [])
return tags if tags is not None else []
return []

8
litellm/types/router.py
View file

@ -902,6 +902,14 @@ class TaggedPreRoutingStrategy(Generic[_PreRoutingStrategyT_co]):
strategy: _PreRoutingStrategyT_co
@dataclass(frozen=True, slots=True)
class ConsumedRequestTagsStamp:
"""The model group a tagged router rewrote to, plus the request tags spent selecting it."""
model_group: str
tags: tuple[str, ...]
@runtime_checkable
class PreRoutingStrategy(Protocol):
"""Structural interface shared by the auto / complexity / adaptive / quality routers."""

8
tests/test_litellm/router_strategy/test_complexity_router.py
View file

@ -1157,8 +1157,8 @@ class TestPreRoutingStrategyRegistry:
TaggedPreRoutingStrategy(tags=("us",), strategy=us),
]
}
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["us"]}}) is us
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["cn"]}}) is cn
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["us"]}}).strategy is us
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["cn"]}}).strategy is cn
assert router._select_pre_routing_strategy("missing", {"metadata": {"tags": ["cn"]}}) is None
router.complexity_routers = {
@ -1167,14 +1167,14 @@ class TestPreRoutingStrategyRegistry:
TaggedPreRoutingStrategy(tags=("default",), strategy=fallback),
]
}
assert router._select_pre_routing_strategy("smart", {}) is fallback
assert router._select_pre_routing_strategy("smart", {}).strategy is fallback
router.complexity_routers = {
"smart": [
TaggedPreRoutingStrategy(tags=("cn",), strategy=cn),
TaggedPreRoutingStrategy(tags=("us",), strategy=us),
]
}
assert router._select_pre_routing_strategy("smart", {}) is cn
assert router._select_pre_routing_strategy("smart", {}).strategy is cn
class TestAsyncPreRoutingHookMultiFormat:

242
tests/test_litellm/router_strategy/test_router_tag_routing.py
View file

@ -2810,3 +2810,245 @@ def test_update_router_config_schema_includes_tag_routing_prefix():
config = UpdateRouterConfig(tag_routing_prefix="route:")
assert config.model_dump(exclude_none=True)["tag_routing_prefix"] == "route:"
# --- issue #36621: the request tags that selected a tagged pre-routing strategy
# (e.g. an auto_router marker) are consumed by that selection and must not
# re-apply to the routed tier's model group; key/team-inherited constraints
# must keep applying there ---
class _RewriteToTierStrategy:
def __init__(self, rewrite_to: str):
self.rewrite_to = rewrite_to
async def async_pre_routing_hook(
self, model, request_kwargs, messages=None, input=None, specific_deployment=False
):
from litellm.types.router import PreRoutingHookResponse
return PreRoutingHookResponse(model=self.rewrite_to, messages=messages)
def _tagged_marker_router(tier_tags=None):
from litellm.types.router import TaggedPreRoutingStrategy
tier_params = {"model": "gemini/gemini-3.6-flash"}
if tier_tags is not None:
tier_params["tags"] = tier_tags
router = litellm.Router(
model_list=[
{
"model_name": "gpt4o",
"litellm_params": {"model": "openai/gpt-4o"},
"model_info": {"id": "plain-gpt4o"},
},
{
"model_name": "gemini-flash",
"litellm_params": tier_params,
"model_info": {"id": "tier-gemini-flash"},
},
],
enable_tag_filtering=True,
)
router.auto_routers = {
"gpt4o": [TaggedPreRoutingStrategy(tags=("route",), strategy=_RewriteToTierStrategy("gemini-flash"))]
}
return router
@pytest.mark.asyncio()
async def test_router_selecting_tag_is_not_reapplied_to_the_routed_tier():
# The exact request the auto-router exists to serve: tags=["route"] selects
# the tagged marker, the strategy rewrites to gemini-flash, and the untagged
# tier deployment must serve it instead of 401ing on the already-spent tag.
router = _tagged_marker_router()
response = await router.acompletion(
model="gpt4o",
messages=[{"role": "user", "content": "What is the capital of France?"}],
metadata={"tags": ["route"], "inherited_tags": []},
mock_response="Paris",
)
assert response._hidden_params["model_id"] == "tier-gemini-flash"
@pytest.mark.asyncio()
async def test_router_selecting_tag_is_consumed_on_litellm_metadata_shaped_requests():
# /v1/messages (and other litellm_metadata endpoints) store proxy metadata,
# including x-litellm-tags header tags, under "litellm_metadata"; consumption
# must read and stamp that same bucket instead of only "metadata".
router = _tagged_marker_router()
deployment = await router.async_get_available_deployment(
model="gpt4o",
request_kwargs={"litellm_metadata": {"tags": ["route"], "inherited_tags": []}},
messages=[{"role": "user", "content": "What is the capital of France?"}],
)
assert deployment["model_info"]["id"] == "tier-gemini-flash"
def test_consumed_request_tags_stamp_names_the_routed_group_and_spent_tags_only_on_a_tag_match():
from litellm.types.router import ConsumedRequestTagsStamp, PreRoutingHookResponse
router = _tagged_marker_router()
strategy = router.auto_routers["gpt4o"][0]
rewrite = PreRoutingHookResponse(model="gemini-flash", messages=None)
consumed = router._consumed_request_tags_stamp(
selected_strategy=strategy, pre_routing_hook_response=rewrite, request_tags=["route"]
)
unmatched = router._consumed_request_tags_stamp(
selected_strategy=strategy, pre_routing_hook_response=rewrite, request_tags=["other"]
)
assert consumed == ConsumedRequestTagsStamp(model_group="gemini-flash", tags=("route",))
assert unmatched is None
@pytest.mark.asyncio()
async def test_tagged_request_direct_to_plain_group_still_rejected():
# Sent straight to the tier, no router selection consumed the tag, so strict
# tag filtering must reject exactly as before.
router = _tagged_marker_router()
with pytest.raises(Exception) as exc_info:
await router.acompletion(
model="gemini-flash",
messages=[{"role": "user", "content": "hi"}],
metadata={"tags": ["route"], "inherited_tags": []},
mock_response="hi",
)
from litellm.types.router import RouterErrors
assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value)
@pytest.mark.asyncio()
async def test_caller_forged_consumption_stamp_is_neutralized_by_the_hook():
# A caller pre-loading the stamp in metadata must not unlock a plain group:
# the pre-routing hook writes-or-clears the stamp on every attempt, and this
# group has no registered strategy, so the forged value is cleared before
# tag filtering runs.
router = _tagged_marker_router()
with pytest.raises(Exception) as exc_info:
await router.acompletion(
model="gemini-flash",
messages=[{"role": "user", "content": "hi"}],
metadata={
"tags": ["route"],
"inherited_tags": [],
"_consumed_request_tags": {"model_group": "gemini-flash", "tags": ["route"]},
},
mock_response="hi",
)
from litellm.types.router import RouterErrors
assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value)
@pytest.mark.asyncio()
async def test_inherited_constraint_still_applies_to_the_routed_tier():
# &region:eu comes from key/team policy (present in inherited_tags):
# consuming the router-selecting "route" tag must not also discard the
# inherited requirement, so a tier without the tag still raises...
with pytest.raises(Exception) as exc_info:
await _tagged_marker_router().acompletion(
model="gpt4o",
messages=[{"role": "user", "content": "hi"}],
metadata={"tags": ["route", "&region:eu"], "inherited_tags": ["&region:eu"]},
mock_response="hi",
)
from litellm.types.router import RouterErrors
assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value)
# ...and a tier carrying it serves the request even though it lacks "route".
response = await _tagged_marker_router(tier_tags=["region:eu"]).acompletion(
model="gpt4o",
messages=[{"role": "user", "content": "hi"}],
metadata={"tags": ["route", "&region:eu"], "inherited_tags": ["&region:eu"]},
mock_response="hi",
)
assert response._hidden_params["model_id"] == "tier-gemini-flash"
def test_request_tags_after_router_consumption_scopes_to_the_stamped_group():
from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY
from litellm.router_strategy.tag_based_routing import _request_tags_after_router_consumption
from litellm.types.router import ConsumedRequestTagsStamp
metadata = {
"tags": ["route", "&region:eu"],
"inherited_tags": ["&region:eu"],
CONSUMED_REQUEST_TAGS_METADATA_KEY: ConsumedRequestTagsStamp(model_group="gemini-flash", tags=("route",)),
}
assert _request_tags_after_router_consumption(metadata, "gemini-flash") == ("&region:eu",)
assert _request_tags_after_router_consumption(metadata, "other-group") == ["route", "&region:eu"]
def test_request_tags_after_router_consumption_drops_only_the_consumed_tags():
from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY
from litellm.router_strategy.tag_based_routing import _request_tags_after_router_consumption
from litellm.types.router import ConsumedRequestTagsStamp
fully_consumed = {
"tags": ["route"],
CONSUMED_REQUEST_TAGS_METADATA_KEY: ConsumedRequestTagsStamp(model_group="gemini-flash", tags=("route",)),
}
assert _request_tags_after_router_consumption(fully_consumed, "gemini-flash") is None
partially_consumed = {
"tags": ["route", "deploy:us"],
"inherited_tags": [],
CONSUMED_REQUEST_TAGS_METADATA_KEY: ConsumedRequestTagsStamp(model_group="gemini-flash", tags=("route",)),
}
assert _request_tags_after_router_consumption(partially_consumed, "gemini-flash") == ("deploy:us",)
@pytest.mark.asyncio()
async def test_non_router_tags_still_pick_the_matching_tier_deployment():
# tags=["route", "deploy:us"]: "route" picks the router and is spent there,
# but "deploy:us" must keep constraining deployment choice inside the routed
# group instead of being dropped with it.
from litellm.types.router import TaggedPreRoutingStrategy
router = litellm.Router(
model_list=[
{
"model_name": "gpt4o",
"litellm_params": {"model": "openai/gpt-4o"},
"model_info": {"id": "plain-gpt4o"},
},
{
"model_name": "gemini-flash",
"litellm_params": {"model": "gemini/gemini-3.6-flash", "tags": ["deploy:us"]},
"model_info": {"id": "tier-gemini-flash-us"},
},
{
"model_name": "gemini-flash",
"litellm_params": {"model": "gemini/gemini-3.6-flash", "tags": ["deploy:eu"]},
"model_info": {"id": "tier-gemini-flash-eu"},
},
],
enable_tag_filtering=True,
)
router.auto_routers = {
"gpt4o": [TaggedPreRoutingStrategy(tags=("route",), strategy=_RewriteToTierStrategy("gemini-flash"))]
}
response = await router.acompletion(
model="gpt4o",
messages=[{"role": "user", "content": "hi"}],
metadata={"tags": ["route", "deploy:us"], "inherited_tags": []},
mock_response="hi",
)
assert response._hidden_params["model_id"] == "tier-gemini-flash-us"

96
tests/test_litellm/test_router.py
View file

@ -7475,6 +7475,102 @@ def test_pre_call_checks_keeps_deployment_when_provider_is_unresolvable(monkeypa
assert len(result) == 1
class TestConsumedRequestTagsStamp:
"""Issue #36621: when a request's tags select a tagged pre-routing strategy, those
tags are consumed by the selection; the hook must stamp the rewritten model group so
tag filtering skips request-body tags there, and must clear the stamp on every
re-entry (fallbacks reuse the same request_kwargs) so it cannot leak elsewhere."""
class _RewriteStrategy:
def __init__(self, rewrite_to: str):
self.rewrite_to = rewrite_to
async def async_pre_routing_hook(
self, model, request_kwargs, messages=None, input=None, specific_deployment=False
):
from litellm.types.router import PreRoutingHookResponse
return PreRoutingHookResponse(model=self.rewrite_to, messages=messages)
@classmethod
def _router(cls, marker_tags=("route",)) -> "litellm.Router":
from litellm.types.router import TaggedPreRoutingStrategy
router = litellm.Router(
model_list=[
{"model_name": "gpt4o", "litellm_params": {"model": "openai/gpt-4o"}},
{"model_name": "gemini-flash", "litellm_params": {"model": "gemini/gemini-3.6-flash"}},
],
enable_tag_filtering=True,
)
router.auto_routers = {
"gpt4o": [TaggedPreRoutingStrategy(tags=marker_tags, strategy=cls._RewriteStrategy("gemini-flash"))]
}
return router
@pytest.mark.asyncio
async def test_stamps_the_rewritten_group_when_request_tags_selected_the_router(self):
from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY
from litellm.types.router import ConsumedRequestTagsStamp
router = self._router()
request_kwargs = {"metadata": {"tags": ["route"]}}
await router.async_pre_routing_hook(model="gpt4o", request_kwargs=request_kwargs)
assert request_kwargs["metadata"][CONSUMED_REQUEST_TAGS_METADATA_KEY] == ConsumedRequestTagsStamp(
model_group="gemini-flash", tags=("route",)
)
@pytest.mark.asyncio
async def test_stamps_into_litellm_metadata_when_the_request_uses_that_bucket(self):
from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY
from litellm.types.router import ConsumedRequestTagsStamp
router = self._router()
request_kwargs = {"litellm_metadata": {"tags": ["route"]}}
await router.async_pre_routing_hook(model="gpt4o", request_kwargs=request_kwargs)
assert request_kwargs["litellm_metadata"][CONSUMED_REQUEST_TAGS_METADATA_KEY] == ConsumedRequestTagsStamp(
model_group="gemini-flash", tags=("route",)
)
@pytest.mark.asyncio
async def test_fallback_reentry_with_a_plain_group_clears_the_stale_stamp(self):
from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY
router = self._router()
request_kwargs = {"metadata": {"tags": ["route"]}}
await router.async_pre_routing_hook(model="gpt4o", request_kwargs=request_kwargs)
await router.async_pre_routing_hook(model="gemini-flash", request_kwargs=request_kwargs)
assert CONSUMED_REQUEST_TAGS_METADATA_KEY not in request_kwargs["metadata"]
@pytest.mark.asyncio
async def test_no_stamp_when_the_request_is_untagged(self):
from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY
router = self._router()
request_kwargs = {"metadata": {}}
await router.async_pre_routing_hook(model="gpt4o", request_kwargs=request_kwargs)
assert CONSUMED_REQUEST_TAGS_METADATA_KEY not in request_kwargs["metadata"]
@pytest.mark.asyncio
async def test_no_stamp_when_the_selected_strategy_carries_no_tags(self):
from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY
router = self._router(marker_tags=())
request_kwargs = {"metadata": {"tags": ["route"]}}
await router.async_pre_routing_hook(model="gpt4o", request_kwargs=request_kwargs)
assert CONSUMED_REQUEST_TAGS_METADATA_KEY not in request_kwargs["metadata"]
class TestAutoRouterMaxInputCharsWiring:
"""`auto_router_max_input_chars` on the deployment has to reach the AutoRouter that embeds prompts.

2
type-discipline-budget.json
View file

@ -1,6 +1,6 @@
{
"LIT001": {
"limit": 23003
"limit": 23001
},
"LIT002": {
"limit": 27146