diff --git a/litellm/constants.py b/litellm/constants.py index 8a6a044ff7b..82ca92475ec 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1464,6 +1464,7 @@ SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affin OUTPUT_TOKEN_CEILING_PARAMS: Final = frozenset({"max_tokens", "max_completion_tokens", "max_output_tokens"}) CLIENT_OUTPUT_CEILING_METADATA_KEY: Final = "_client_output_ceiling" CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags" +ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated" SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted" diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index d9d56778405..770963a1f24 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -13,6 +13,7 @@ from litellm.constants import ( CLIENT_OUTPUT_CEILING_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, PRE_CALL_EXECUTED_GUARDRAILS_KEY, + ROUTING_REQUEST_TAGS_METADATA_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, ) from litellm.integrations.custom_logger import CustomLogger @@ -509,6 +510,7 @@ LITELLM_PROXY_INTERNAL_METADATA_KEYS: Final = frozenset( SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, CLIENT_OUTPUT_CEILING_METADATA_KEY, + ROUTING_REQUEST_TAGS_METADATA_KEY, "disable_global_guardrails", "disable_global_guardrail", "opted_out_global_guardrails", diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 893174dab03..f81fe312645 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -24,6 +24,7 @@ from litellm.constants import ( LITELLM_PROXY_MASTER_KEY_ALIAS, OTEL_SERVICE_NAME_METADATA_KEYS, PRE_CALL_EXECUTED_GUARDRAILS_KEY, + ROUTING_REQUEST_TAGS_METADATA_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, @@ -290,6 +291,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = ( GATEWAY_INJECTED_CACHE_METADATA_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, + ROUTING_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, "standard_logging_object", "proxy_server_request", diff --git a/litellm/router.py b/litellm/router.py index 9b2badeaa69..b0a1ddd164e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -62,6 +62,7 @@ from litellm.constants import ( DEFAULT_MAX_LRU_CACHE_SIZE, INTERNAL_CALL_ORIGIN_METADATA_KEY, OUTPUT_TOKEN_CEILING_PARAMS, + ROUTING_REQUEST_TAGS_METADATA_KEY, RUNTIME_UPDATABLE_ROUTER_SETTINGS, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, ) @@ -3772,6 +3773,11 @@ class Router: refund_stale_reservation_before_retry(self.cache, kwargs) set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=deployment_has_io_token_limits(deployment)) + kwargs[metadata_variable_name].setdefault( + ROUTING_REQUEST_TAGS_METADATA_KEY, + tuple(_get_tags_from_request_kwargs(kwargs, metadata_variable_name=metadata_variable_name)), + ) + ## DEPLOYMENT-LEVEL TAGS deployment_tags: Final = deployment.get("litellm_params", {}).get("tags") if deployment_tags: diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index eabd9278cf6..d4f46e94579 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -13,7 +13,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload from litellm._logging import verbose_logger -from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY +from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs from litellm.types.router import ConsumedRequestTagsStamp, DeploymentTypedDict, RouterErrors @@ -461,7 +461,10 @@ def _request_tags_after_router_consumption(metadata: object, model: str) -> Sequ if not isinstance(metadata, Mapping): return None typed_metadata: Final[Mapping[str, object]] = metadata - request_tags: Final = _tags_in_metadata(typed_metadata) + request_tags: Final = _tags_in_metadata( + typed_metadata, + key=ROUTING_REQUEST_TAGS_METADATA_KEY if ROUTING_REQUEST_TAGS_METADATA_KEY in typed_metadata else "tags", + ) stamp: Final = typed_metadata.get(CONSUMED_REQUEST_TAGS_METADATA_KEY) if not isinstance(stamp, ConsumedRequestTagsStamp) or stamp.model_group != model: return request_tags @@ -646,7 +649,7 @@ async def get_deployments_for_tag( return healthy_deployments -def _tags_in_metadata(metadata: object) -> list[str]: +def _tags_in_metadata(metadata: object, key: str = "tags") -> list[str]: """ Tags out of a metadata bucket the caller controls the shape of. @@ -657,7 +660,7 @@ def _tags_in_metadata(metadata: object) -> list[str]: if not isinstance(metadata, Mapping): return [] typed_metadata: Final[Mapping[str, object]] = metadata - tags: Final = typed_metadata.get("tags") + tags: Final = typed_metadata.get(key) if isinstance(tags, str) or not isinstance(tags, Sequence): return [] typed_tags: Final[Sequence[object]] = tags diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py index 59a59c7e16d..16c641b8d29 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py @@ -3031,6 +3031,21 @@ def test_request_tags_after_router_consumption_drops_only_the_consumed_tags(): assert _request_tags_after_router_consumption(partially_consumed, "gemini-flash") == ("deploy:us",) +def test_request_tags_after_router_consumption_ignores_tags_merged_from_prior_deployments(): + from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY + from litellm.router_strategy.tag_based_routing import _request_tags_after_router_consumption + from litellm.types.router import ConsumedRequestTagsStamp + + metadata = { + "tags": ["route", "®ion:eu", "free"], + ROUTING_REQUEST_TAGS_METADATA_KEY: ("route", "®ion:eu"), + "inherited_tags": ["®ion:eu"], + CONSUMED_REQUEST_TAGS_METADATA_KEY: ConsumedRequestTagsStamp(model_group="gemini-flash", tags=("route",)), + } + assert _request_tags_after_router_consumption(metadata, "gemini-flash") == ("®ion:eu",) + assert _request_tags_after_router_consumption(metadata, "other-group") == ["route", "®ion:eu"] + + @pytest.mark.asyncio() async def test_non_router_tags_still_pick_the_matching_tier_deployment(): # tags=["route", "deploy:us"]: "route" picks the router and is spent there, diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index eed34c79a06..29df6df0477 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -40,7 +40,7 @@ from litellm.router import ( _is_retriable_anthropic_status, ) from litellm.router_strategy import simple_shuffle -from litellm.types.router import DeploymentTypedDict +from litellm.types.router import DeploymentTypedDict, RetryPolicy def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata(): @@ -6132,6 +6132,50 @@ def test_update_kwargs_with_deployment_no_tags(): assert "tags" not in kwargs["metadata"] +@pytest.mark.asyncio +async def test_retry_does_not_narrow_tag_filtered_group_to_failed_deployments_tags(): + router = Router( + model_list=[ + { + "model_name": "tagged-group", + "litellm_params": { + "model": "openai/gpt-5.5", + "api_key": "fake-key", + "tags": ["free"], + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000001, + "mock_response": "litellm.ContextWindowExceededError", + }, + "model_info": {"id": "tagged-failing"}, + }, + { + "model_name": "tagged-group", + "litellm_params": { + "model": "openai/gpt-5.5", + "api_key": "fake-key", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.001, + "mock_response": "ok", + }, + "model_info": {"id": "untagged-healthy"}, + }, + ], + routing_strategy="cost-based-routing", + enable_tag_filtering=True, + num_retries=2, + retry_after=0, + retry_policy=RetryPolicy(BadRequestErrorRetries=2), + ) + metadata: Final[dict[str, object]] = {} + + response = await router.acompletion( + model="tagged-group", messages=[{"role": "user", "content": "hi"}], metadata=metadata + ) + + assert response._hidden_params["model_id"] == "untagged-healthy" + assert metadata["tags"] == ["free"] + + def test_update_kwargs_with_deployment_merges_tools(): """ Test that when both deployment litellm_params and request have tools,