diff --git a/litellm/router.py b/litellm/router.py index 20af2e5649e..f2aa2177f79 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -118,6 +118,7 @@ from litellm.litellm_core_utils.batch_utils import ( ) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, coerce_token_limit, get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, @@ -264,6 +265,7 @@ from litellm.router_utils.common_utils import ( format_no_fallback_group_message, get_request_team_id, is_proxy_admin_request, + metadata_defaults_under_request, provider_for_generic_call, request_user_api_key_auth, resolve_model_group_alias, @@ -677,6 +679,7 @@ _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None) _ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) _FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_REQUEST_METADATA_ADAPTER: Final = TypeAdapter(Mapping[object, object]) _DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLiteLLMParams]] = TypeAdapter( Mapping[str, object] | GenericLiteLLMParams ) @@ -5037,7 +5040,7 @@ class Router(DeprecatedRouterMembers): """ # 1) copy your defaults and pull out metadata defaults: Final = dict(without_retrieve_stream_defaults(self._core_default_litellm_params, kwargs)) - metadata_defaults: Final = defaults.pop("metadata", {}) or {} + metadata_defaults: Final = as_str_mapping(defaults.pop("metadata", None)) or {} # 2) add any non-metadata defaults that aren't already in kwargs for key, value in defaults.items(): @@ -5046,7 +5049,12 @@ class Router(DeprecatedRouterMembers): kwargs.setdefault(key, value) # 3) merge in metadata, this handles inserting this as either "metadata" or "litellm_metadata" - kwargs.setdefault(metadata_variable_name, {}).update(metadata_defaults) + request_metadata: Final = _REQUEST_METADATA_ADAPTER.validate_python( + kwargs.setdefault(metadata_variable_name, {}) + ) + kwargs[metadata_variable_name].update( + metadata_defaults_under_request(defaults=metadata_defaults, request_metadata=request_metadata) + ) def _handle_clientside_credential( self, deployment: dict, kwargs: dict, function_name: str | None = None diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 2451422b5e6..f4c4ad466a2 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -13,6 +13,7 @@ import litellm from litellm._logging import verbose_logger, verbose_router_logger from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure @@ -61,6 +62,32 @@ def without_router_only_kwargs(kwargs: Mapping[str, _V]) -> dict[str, _V]: return {key: value for key, value in kwargs.items() if key not in ROUTER_ONLY_CALL_KWARGS} +_ROUTER_OWNED_METADATA_KEYS: Final = frozenset({"caching_groups"}) + + +def metadata_defaults_under_request( + defaults: Mapping[str, object], request_metadata: Mapping[object, object] +) -> Mapping[str, object]: + return { + key: ( + default + if key in _ROUTER_OWNED_METADATA_KEYS + else _request_value_over_default(default=default, requested=request_metadata.get(key)) + ) + for key, default in defaults.items() + } + + +def _request_value_over_default(default: object, requested: object) -> object: + if requested is None: + return default + default_mapping: Final = as_str_mapping(default) + requested_mapping: Final = as_str_mapping(requested) + if default_mapping is None or requested_mapping is None: + return requested + return {**default_mapping, **requested_mapping} + + def as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]: if not isinstance(value, tuple): return () diff --git a/tests/integration/providers/test_vertex_labels_router_default_metadata_wire.py b/tests/integration/providers/test_vertex_labels_router_default_metadata_wire.py new file mode 100644 index 00000000000..89747a5bed8 --- /dev/null +++ b/tests/integration/providers/test_vertex_labels_router_default_metadata_wire.py @@ -0,0 +1,85 @@ +import json +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +_MODEL_NAME: Final = "labelled-gemini" +_PROJECT: Final = "scripted-project" +_LOCATION: Final = "us-central1" +_ROUTER_LABELS: Final = {"env": "prod"} +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_REPLY: Final = json.dumps( + { + "candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 1, "totalTokenCount": 4}, + } +).encode() + + +def _gemini_peer(request: Request) -> Reply: + if request.target.endswith(":generateContent"): + return Reply(body=_REPLY) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _config(directory: Path, gateway: Gateway, api_base: str) -> Path: + config: Final = directory / "router-default-requester-metadata.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": _MODEL_NAME, + "litellm_params": { + "model": "vertex_ai/gemini-3.8-flash", + "api_base": api_base, + "vertex_project": _PROJECT, + "vertex_location": _LOCATION, + "vertex_credentials": service_account_json(_PROJECT, gateway.upstream_url.rstrip("/")), + }, + } + ], + "router_settings": {"default_litellm_params": {"metadata": {"requester_metadata": _ROUTER_LABELS}}}, + } + ), + encoding="utf-8", + ) + return config + + +def _labels_sent(proxy: Gateway, wire: Wire, metadata: Mapping[str, JsonValue] | None) -> JsonValue: + response: Final = proxy.request( + "POST", + "/v1/chat/completions", + { + "model": _MODEL_NAME, + "messages": [{"role": "user", "content": "Return exactly OK."}], + **({"metadata": dict(metadata)} if metadata is not None else {}), + }, + ) + assert response.status_code == 200, response.text + generations: Final = tuple(request for request in wire.drain() if request.target.endswith(":generateContent")) + assert len(generations) == 1, generations + return _JSON_OBJECT.validate_json(generations[0].body).get("labels") + + +def test_router_default_requester_metadata_merges_under_each_callers_vertex_labels( + gateway: Gateway, tmp_path: Path +) -> None: + with ( + wire_server(_gemini_peer) as wire, + owned_proxy_process(gateway, tmp_path, {}, config=_config(tmp_path, gateway, wire.url)) as owned, + ): + proxy: Final = Gateway(owned.gateway.client, owned.gateway.key, gateway.upstream_url) + assert _labels_sent(proxy, wire, {"team": "ai-platform"}) == {**_ROUTER_LABELS, "team": "ai-platform"} + assert _labels_sent(proxy, wire, {"env": "staging"}) == {"env": "staging"} + assert _labels_sent(proxy, wire, None) == _ROUTER_LABELS diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 5f1b6032ca8..d5cc8fa7e92 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -20,6 +20,7 @@ from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook from litellm.types.litellm_params import RoutingStrategyName from litellm.types.router import AlertingConfig, Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo @@ -396,6 +397,80 @@ def test_update_kwargs_with_default_litellm_params(model_list): assert kwargs["metadata"]["key2"] == "value2" +def _vertex_labels_router(metadata_defaults: Mapping[str, object]) -> Router: + return Router( + model_list=[{"model_name": "g", "litellm_params": {"model": "vertex_ai/gemini-3.8-flash"}}], + default_litellm_params={"metadata": dict(metadata_defaults)}, + ) + + +def _vertex_labels_after_defaults(router: Router, request_metadata: dict, metadata_variable_name: str): + kwargs: Final = {metadata_variable_name: request_metadata} + router._update_kwargs_with_default_litellm_params(kwargs=kwargs, metadata_variable_name=metadata_variable_name) + return pop_vertex_request_labels(optional_params={}, litellm_params=kwargs) + + +@pytest.mark.parametrize("metadata_variable_name", ["metadata", "litellm_metadata"]) +@pytest.mark.parametrize( + "request_labels, expected_labels", + [ + ({"litellm-team": "ai-platform"}, {"env": "prod", "litellm-team": "ai-platform"}), + ({"env": "dev", "litellm-team": "ai-platform"}, {"env": "dev", "litellm-team": "ai-platform"}), + (None, {"env": "prod"}), + ({}, {"env": "prod"}), + ], +) +def test_default_requester_metadata_merges_under_request_labels( + metadata_variable_name: str, request_labels: dict | None, expected_labels: dict +): + router: Final = _vertex_labels_router({"requester_metadata": {"env": "prod"}}) + request_metadata: Final = {} if request_labels is None else {"requester_metadata": request_labels} + + labels: Final = _vertex_labels_after_defaults(router, request_metadata, metadata_variable_name) + + assert labels == expected_labels + + +def test_default_requester_metadata_does_not_carry_labels_between_requests(): + router: Final = _vertex_labels_router({"requester_metadata": {"env": "prod"}}) + + first: Final = _vertex_labels_after_defaults(router, {"requester_metadata": {"litellm-team": "a"}}, "metadata") + second: Final = _vertex_labels_after_defaults(router, {}, "metadata") + + assert first == {"env": "prod", "litellm-team": "a"} + assert second == {"env": "prod"} + + +@pytest.mark.parametrize( + "request_metadata, expected_tags", + [ + ({"tags": ["caller"]}, ["caller"]), + ({}, ["proxy-default"]), + ({"tags": None}, ["proxy-default"]), + ], +) +def test_default_metadata_list_value_applies_only_when_request_sets_none(request_metadata: dict, expected_tags: list): + router: Final = _vertex_labels_router({"tags": ["proxy-default"]}) + kwargs: Final = {"metadata": request_metadata} + + router._update_kwargs_with_default_litellm_params(kwargs=kwargs) + + assert kwargs["metadata"]["tags"] == expected_tags + + +@pytest.mark.parametrize("router_caching_groups", [[("g", "other")], None]) +def test_router_caching_groups_win_over_request_metadata(router_caching_groups: list | None): + router: Final = Router( + model_list=[{"model_name": "g", "litellm_params": {"model": "vertex_ai/gemini-3.8-flash"}}], + caching_groups=router_caching_groups, + ) + kwargs: Final = {"metadata": {"caching_groups": [("g", "premium")]}} + + router._update_kwargs_with_default_litellm_params(kwargs=kwargs) + + assert kwargs["metadata"]["caching_groups"] == router_caching_groups + + def test_get_timeout(model_list): """Test if the '_get_timeout' function is working correctly""" router = Router(model_list=model_list)