From b28005fdf66d7e5f810ac1f68d87113b2d9fc9fe Mon Sep 17 00:00:00 2001 From: paveln83 <349044532+paveln83@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:23:52 -0800 Subject: [PATCH] fix(router): merge default metadata under the request's instead of replacing it (#50700) * fix(router): merge default metadata under the request's instead of replacing it * fix(router): join default metadata tags after the request's tags A router default_litellm_params.metadata.tags list replaced the request's tags, so the deployment and credential tags the router adds never reached the logged request tags. Default tags now follow the request's tags, skipping ones already present, like the proxy's key tag merge * fix(router): keep masking a router default turns on Merging default metadata under the request let a caller's mask_input or mask_output false switch off masking an admin set in default_litellm_params.metadata. A masking flag the default turns on now stays on, and a caller can still turn masking on. Also gives this PR's test suppressions a reason and adds a missing blank line * fix(router): keep the router's cache namespace and drop malformed caller tags and dicts A caller's metadata.redis_namespace no longer beats a router default, so it cannot pick another tenant's cache partition. A caller tags value that is not a list, or a non-dict value under a dict default, is treated as absent the way the proxy's merge_tags treats malformed tags, so the router defaults still apply. * test(router): check that mask_output stays on when the router default turns it on * fix(router): keep a router default guardrail whatever guardrails the request lists * fix(router): keep the admin's settings for a router default guardrail when the caller lists it * test(router): check that a router default guardrail entry keeps the admin's settings * fix(router): keep a router default private Langfuse trace private whatever the caller sends * fix(router): keep a router default function such as langfuse_masking_function over the caller's value --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../routing_common_utils.py | 71 +++++ litellm/router_rust/invoke/endpoints.py | 11 +- litellm/router_rust/invoke/shaping.py | 7 +- ...tex_labels_router_default_metadata_wire.py | 85 ++++++ ...st_endpoints_characterization_inference.py | 15 +- .../unit/test_router/core/test_gaps_l2800.py | 259 +++++++++++++++++- 6 files changed, 435 insertions(+), 13 deletions(-) create mode 100644 tests/integration/providers/test_vertex_labels_router_default_metadata_wire.py diff --git a/litellm/litellm_core_utils/routing_common_utils.py b/litellm/litellm_core_utils/routing_common_utils.py index d13ff4059aa..89d6729766f 100644 --- a/litellm/litellm_core_utils/routing_common_utils.py +++ b/litellm/litellm_core_utils/routing_common_utils.py @@ -1,6 +1,7 @@ import hashlib import json from collections.abc import Mapping, Sequence +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Final, TypeVar @@ -13,10 +14,12 @@ import litellm from litellm._logging import verbose_logger, verbose_router_logger from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure from litellm.llms.sagemaker.common_utils import deployment_inference_component +from litellm.secret_managers.main import str_to_bool from litellm.types.router import ( CredentialLiteLLMParams, Deployment, @@ -62,6 +65,74 @@ def without_router_only_kwargs(kwargs: Mapping[str, _V]) -> dict[str, _V]: return {key: value for key, value in kwargs.items() if key not in ROUTER_ONLY_CALL_KWARGS} +_ROUTER_OWNED_METADATA_KEYS: Final = frozenset({"caching_groups", "redis_namespace"}) +_PRIVATE_SETTING_BY_FLAG: Final = MappingProxyType({"mask_input": True, "mask_output": True, "trace_public": False}) +_JOINED_LIST_METADATA_KEYS: Final = frozenset({"tags", "guardrails"}) + + +def metadata_defaults_under_request( + defaults: Mapping[str, object], request_metadata: Mapping[str, object] +) -> Mapping[str, object]: + return { + key: _default_under_request(key=key, default=default, requested=request_metadata.get(key)) + for key, default in defaults.items() + } + + +def _default_under_request(key: str, default: object, requested: object) -> object: + if key in _ROUTER_OWNED_METADATA_KEYS or callable(default): + return default + if _PRIVATE_SETTING_BY_FLAG.get(key) is _is_flag_on(default): + return default + if key in _JOINED_LIST_METADATA_KEYS: + return _request_items_then_default_items(default=default, requested=requested) + return _request_value_over_default(default=default, requested=requested) + + +def _is_flag_on(value: object) -> bool: + parsed: Final = str_to_bool(value) if isinstance(value, str) else None + return bool(value) if parsed is None else parsed + + +def _request_items_then_default_items(default: object, requested: object) -> object: + default_items: Final = as_object_list(default) + if default_items is None: + return _request_value_over_default(default=default, requested=requested) + default_names: Final = frozenset(chain.from_iterable(_entry_names(item) for item in default_items)) + request_items: Final = tuple( + chain.from_iterable(_without_entries_named(item, default_names) for item in as_object_list(requested) or ()) + ) + return [*request_items, *(item for item in default_items if item not in request_items)] + + +def _entry_names(entry: object) -> frozenset[str]: + mapping: Final = as_str_mapping(entry) + if mapping is not None: + return frozenset(mapping) + return frozenset((entry,)) if isinstance(entry, str) else frozenset() + + +def _without_entries_named(entry: object, names: frozenset[str]) -> tuple[object, ...]: + mapping: Final = as_str_mapping(entry) + if mapping is None or names.isdisjoint(mapping): + return (entry,) + if names.issuperset(mapping): + return () + return ({key: value for key, value in mapping.items() if key not in names},) + + +def _request_value_over_default(default: object, requested: object) -> object: + if requested is None: + return default + default_mapping: Final = as_str_mapping(default) + if default_mapping is None: + return requested + requested_mapping: Final = as_str_mapping(requested) + if requested_mapping is None: + return default + return {**default_mapping, **requested_mapping} + + def as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]: if not isinstance(value, tuple): return () diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py index 7a5b52761e2..70c83bed728 100644 --- a/litellm/router_rust/invoke/endpoints.py +++ b/litellm/router_rust/invoke/endpoints.py @@ -2118,7 +2118,10 @@ class RustEndpoints: def text_completion(self, request: Mapping[str, object]) -> object: """The sync `text_completion`: one pick, no retries, the group name as the model.""" - from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin + from litellm.litellm_core_utils.routing_common_utils import ( + deployment_model_id_pin, + metadata_defaults_under_request, + ) kwargs: Final = without(request, "is_retry", "is_fallback", "is_async") model: Final = kwargs.get("model") @@ -2139,9 +2142,9 @@ class RustEndpoints: if key not in kwargs: kwargs[key] = value elif key == "metadata": - bucket = kwargs[key] - if isinstance(bucket, dict): - bucket.update(as_dict(value)) # pyright: ignore[reportUnknownMemberType] # the request's metadata dict + metadata_bucket(kwargs, key).update( + metadata_defaults_under_request(defaults=as_dict(value), request_metadata=as_dict(kwargs[key])) + ) pinned: Final = deployment_model_id_pin(data) if pinned is not None: kwargs["model_id"] = pinned diff --git a/litellm/router_rust/invoke/shaping.py b/litellm/router_rust/invoke/shaping.py index 258f919c9f1..bf6410d8242 100644 --- a/litellm/router_rust/invoke/shaping.py +++ b/litellm/router_rust/invoke/shaping.py @@ -32,7 +32,7 @@ from litellm.litellm_core_utils.core_helpers import without from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, inferred_provider from litellm.litellm_core_utils.request_timeout_resolver import get_service_tier_timeout -from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin +from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin, metadata_defaults_under_request from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy from litellm.responses.retrieve_stream_query import requests_streamed_retrieval, without_retrieve_stream_defaults from litellm.router_api import deployment_id as generate_deployment_id @@ -359,14 +359,15 @@ def update_kwargs_with_default_litellm_params( metadata_variable_name: str, ) -> None: """``_update_kwargs_with_default_litellm_params``: unset non-None defaults fill the request; - the default metadata is merged over the request's.""" + the default metadata fills the keys the request leaves unset, the router's caching groups excepted.""" defaults: Final = dict(without_retrieve_stream_defaults(shaping.default_litellm_params, kwargs)) metadata_defaults: Final = defaults.pop("metadata", {}) or {} # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values for key, value in defaults.items(): if value is None: continue kwargs.setdefault(key, value) - metadata_bucket(kwargs, metadata_variable_name).update(as_dict(metadata_defaults)) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + bucket: Final = metadata_bucket(kwargs, metadata_variable_name) + bucket.update(metadata_defaults_under_request(defaults=as_dict(metadata_defaults), request_metadata=bucket)) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values # --- clientside credentials ------------------------------------------------------------------- diff --git a/tests/integration/providers/test_vertex_labels_router_default_metadata_wire.py b/tests/integration/providers/test_vertex_labels_router_default_metadata_wire.py new file mode 100644 index 00000000000..89747a5bed8 --- /dev/null +++ b/tests/integration/providers/test_vertex_labels_router_default_metadata_wire.py @@ -0,0 +1,85 @@ +import json +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +_MODEL_NAME: Final = "labelled-gemini" +_PROJECT: Final = "scripted-project" +_LOCATION: Final = "us-central1" +_ROUTER_LABELS: Final = {"env": "prod"} +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_REPLY: Final = json.dumps( + { + "candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 1, "totalTokenCount": 4}, + } +).encode() + + +def _gemini_peer(request: Request) -> Reply: + if request.target.endswith(":generateContent"): + return Reply(body=_REPLY) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _config(directory: Path, gateway: Gateway, api_base: str) -> Path: + config: Final = directory / "router-default-requester-metadata.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": _MODEL_NAME, + "litellm_params": { + "model": "vertex_ai/gemini-3.8-flash", + "api_base": api_base, + "vertex_project": _PROJECT, + "vertex_location": _LOCATION, + "vertex_credentials": service_account_json(_PROJECT, gateway.upstream_url.rstrip("/")), + }, + } + ], + "router_settings": {"default_litellm_params": {"metadata": {"requester_metadata": _ROUTER_LABELS}}}, + } + ), + encoding="utf-8", + ) + return config + + +def _labels_sent(proxy: Gateway, wire: Wire, metadata: Mapping[str, JsonValue] | None) -> JsonValue: + response: Final = proxy.request( + "POST", + "/v1/chat/completions", + { + "model": _MODEL_NAME, + "messages": [{"role": "user", "content": "Return exactly OK."}], + **({"metadata": dict(metadata)} if metadata is not None else {}), + }, + ) + assert response.status_code == 200, response.text + generations: Final = tuple(request for request in wire.drain() if request.target.endswith(":generateContent")) + assert len(generations) == 1, generations + return _JSON_OBJECT.validate_json(generations[0].body).get("labels") + + +def test_router_default_requester_metadata_merges_under_each_callers_vertex_labels( + gateway: Gateway, tmp_path: Path +) -> None: + with ( + wire_server(_gemini_peer) as wire, + owned_proxy_process(gateway, tmp_path, {}, config=_config(tmp_path, gateway, wire.url)) as owned, + ): + proxy: Final = Gateway(owned.gateway.client, owned.gateway.key, gateway.upstream_url) + assert _labels_sent(proxy, wire, {"team": "ai-platform"}) == {**_ROUTER_LABELS, "team": "ai-platform"} + assert _labels_sent(proxy, wire, {"env": "staging"}) == {"env": "staging"} + assert _labels_sent(proxy, wire, None) == _ROUTER_LABELS diff --git a/tests/unit/test_router/core/test_endpoints_characterization_inference.py b/tests/unit/test_router/core/test_endpoints_characterization_inference.py index 8db0d6bea12..c975e45b1d3 100644 --- a/tests/unit/test_router/core/test_endpoints_characterization_inference.py +++ b/tests/unit/test_router/core/test_endpoints_characterization_inference.py @@ -205,6 +205,17 @@ async def test_sync_text_completion_merges_default_metadata_into_request_metadat assert (metadata["x"], metadata["y"], metadata["model_group"], sent["user"]) == (1, 2, "g", "u") +async def test_sync_text_completion_keeps_the_request_metadata_over_the_default_metadata( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core( + model_list=[deployment("g", "d1")], + default_litellm_params={"metadata": {"requester_metadata": {"env": "prod", "region": "us"}}}, + ) + core.text_completion(model="g", prompt="p", metadata={"requester_metadata": {"env": "dev", "team": "t1"}}) + assert _bucket(_sent(host))["requester_metadata"] == {"env": "dev", "region": "us", "team": "t1"} + + async def test_sync_text_completion_sends_the_model_group_name_not_the_deployment_model( make_core: MakeCore, host: EndpointHost ): @@ -329,12 +340,12 @@ async def test_default_litellm_params_metadata_merges_into_request_metadata(make assert (metadata["team"], metadata["trace"], metadata["model_group"]) == ("t", "x", "g") -async def test_default_litellm_params_metadata_overwrites_the_same_request_metadata_key( +async def test_the_request_metadata_key_wins_over_the_same_default_litellm_params_metadata_key( make_core: MakeCore, host: EndpointHost ): core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"team": "t"}}) await core.acompletion(model="g", messages=MESSAGES, metadata={"team": "request"}) - assert _bucket(_sent(host))["team"] == "t" + assert _bucket(_sent(host))["team"] == "request" async def test_default_litellm_params_metadata_goes_to_litellm_metadata_for_generic_endpoints( diff --git a/tests/unit/test_router/core/test_gaps_l2800.py b/tests/unit/test_router/core/test_gaps_l2800.py index 78e237f0696..196ce20d4b3 100644 --- a/tests/unit/test_router/core/test_gaps_l2800.py +++ b/tests/unit/test_router/core/test_gaps_l2800.py @@ -246,7 +246,7 @@ async def test_responses_replace_kubernetes_pod_routing_in_the_request_metadata( """On the Responses path the routing goes to ``litellm_metadata``; a pod routing key the caller put in ``metadata`` is overwritten with the serving deployment's (none here).""" core: Final = make_core(model_list=[deployment("g", "d1")]) - await core.aresponses( # pyright: ignore[reportAttributeAccessIssue] + await core.aresponses( # pyright: ignore[reportAttributeAccessIssue] # endpoint method model="g", input="hi", metadata={KUBERNETES_POD_ROUTING_KEY: {"pod": "stale"}, "keep": 1} ) sent: Final = host.calls[0].kwargs @@ -332,7 +332,7 @@ async def test_the_process_request_timeout_outranks_the_router_timeout_once_a_ro process settings), and only when the router has a ``timeout`` of its own.""" monkeypatch.setattr(litellm, "request_timeout", 33.0) core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42) - await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # endpoint method assert host.calls[0].kwargs["timeout"] == 33.0 @@ -353,7 +353,7 @@ async def test_default_litellm_params_timeout_applies_when_no_router_or_process_ ) -> None: host.set_dynamic(request_timeout=None) core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"timeout": 9}) - await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # endpoint method assert host.calls[0].kwargs["timeout"] == 9 @@ -368,6 +368,257 @@ async def test_default_litellm_params_stream_timeout_is_the_last_stream_timeout_ assert host.calls[0].kwargs["stream_timeout"] == 4 +_PROXY_REQUESTER_METADATA: Final = {"metadata": {"requester_metadata": {"env": "prod"}}} + + +@pytest.mark.parametrize( + ("request_metadata", "sent_requester_metadata"), + [ + ({"requester_metadata": {"team": "t1"}}, {"env": "prod", "team": "t1"}), + ({"requester_metadata": {"env": "dev", "team": "t1"}}, {"env": "dev", "team": "t1"}), + ({"requester_metadata": None}, {"env": "prod"}), + ({"requester_metadata": {}}, {"env": "prod"}), + ({"requester_metadata": "x"}, {"env": "prod"}), + ({}, {"env": "prod"}), + ], +) +async def test_default_requester_metadata_merges_under_the_request_requester_metadata( + make_core: MakeCore, + host: ScriptedHost, + request_metadata: Mapping[str, object], + sent_requester_metadata: Mapping[str, str], +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params=_PROXY_REQUESTER_METADATA) + await core.acompletion(model="g", messages=MESSAGES, metadata=dict(request_metadata)) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["requester_metadata"] == sent_requester_metadata + + +async def test_responses_default_requester_metadata_merges_under_the_request_litellm_metadata( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params=_PROXY_REQUESTER_METADATA) + await core.aresponses( # pyright: ignore[reportAttributeAccessIssue] # endpoint method + model="g", input="hi", litellm_metadata={"requester_metadata": {"team": "t1"}} + ) + sent: Final = _metadata(host.calls[0].kwargs, "litellm_metadata") + assert sent["requester_metadata"] == {"env": "prod", "team": "t1"} + + +async def test_default_requester_metadata_does_not_carry_one_request_labels_into_the_next( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params=_PROXY_REQUESTER_METADATA) + await core.acompletion(model="g", messages=MESSAGES, metadata={"requester_metadata": {"team": "t1"}}) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[1].kwargs)["requester_metadata"] == {"env": "prod"} + + +@pytest.mark.parametrize( + ("request_metadata", "sent_regions"), + [({"regions": ["eu"]}, ["eu"]), ({"regions": None}, ["us"]), ({}, ["us"])], +) +async def test_a_default_metadata_list_applies_only_when_the_request_sets_none( + make_core: MakeCore, host: ScriptedHost, request_metadata: Mapping[str, object], sent_regions: list[str] +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"regions": ["us"]}} + ) + await core.acompletion(model="g", messages=MESSAGES, metadata=dict(request_metadata)) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["regions"] == sent_regions + + +@pytest.mark.parametrize("mask_key", ["mask_input", "mask_output"]) +@pytest.mark.parametrize( + ("default_mask", "request_mask", "sent_mask"), + [ + (True, False, True), + (True, None, True), + ("true", False, "true"), + (False, True, True), + (False, None, False), + ("false", True, True), + ], +) +async def test_default_metadata_masking_stays_on_once_either_side_turns_it_on( + make_core: MakeCore, + host: ScriptedHost, + mask_key: str, + default_mask: object, + request_mask: object, + sent_mask: object, +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {mask_key: default_mask}} + ) + await core.acompletion(model="g", messages=MESSAGES, metadata={mask_key: request_mask}) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)[mask_key] == sent_mask + + +@pytest.mark.parametrize( + ("default_public", "request_public", "sent_public"), + [ + (False, True, False), + ("false", True, "false"), + (False, None, False), + (True, False, False), + (True, None, True), + ("true", True, True), + ], +) +async def test_a_private_default_trace_stays_private_whatever_the_request_sends( + make_core: MakeCore, + host: ScriptedHost, + default_public: object, + request_public: object, + sent_public: object, +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"trace_public": default_public}} + ) + await core.acompletion(model="g", messages=MESSAGES, metadata={"trace_public": request_public}) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["trace_public"] == sent_public + + +async def test_a_default_redis_namespace_beats_the_request_redis_namespace( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"redis_namespace": "tenant-a"}} + ) + await core.acompletion(model="g", messages=MESSAGES, metadata={"redis_namespace": "tenant-b"}) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["redis_namespace"] == "tenant-a" + + +def _redact(_value: object) -> str: + return "redacted" + + +@pytest.mark.parametrize("request_value", ["", {"caller": "value"}]) +async def test_a_default_masking_function_beats_whatever_the_request_sends( + make_core: MakeCore, host: ScriptedHost, request_value: object +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"langfuse_masking_function": _redact}} + ) + await core.acompletion(model="g", messages=MESSAGES, metadata={"langfuse_masking_function": request_value}) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["langfuse_masking_function"] is _redact + + +_PROXY_DEFAULT_TAGS: Final = {"metadata": {"tags": ["proxy-default"]}} + + +@pytest.mark.parametrize( + ("request_metadata", "sent_tags"), + [ + ({"tags": ["caller"]}, ["caller", "proxy-default"]), + ({"tags": ["proxy-default", "caller"]}, ["proxy-default", "caller"]), + ({"tags": None}, ["proxy-default"]), + ({}, ["proxy-default"]), + ({"tags": "caller"}, ["proxy-default"]), + ], +) +async def test_default_metadata_tags_follow_the_request_tags( + make_core: MakeCore, host: ScriptedHost, request_metadata: Mapping[str, object], sent_tags: object +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params=_PROXY_DEFAULT_TAGS) + await core.acompletion(model="g", messages=MESSAGES, metadata=dict(request_metadata)) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["tags"] == sent_tags + + +@pytest.mark.parametrize( + ("request_metadata", "sent_tags"), + [ + ({"tags": ["caller"]}, ["caller"]), + ({}, "proxy-default"), + ], +) +async def test_a_default_tags_that_is_not_a_list_yields_to_the_request_tags( + make_core: MakeCore, host: ScriptedHost, request_metadata: Mapping[str, object], sent_tags: object +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"tags": "proxy-default"}} + ) + await core.acompletion(model="g", messages=MESSAGES, metadata=dict(request_metadata)) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["tags"] == sent_tags + + +@pytest.mark.parametrize( + ("request_metadata", "sent_tags"), + [ + ({}, ["t1", "Credential: cred", "proxy-default"]), + ({"tags": ["caller"]}, ["caller", "t1", "Credential: cred", "proxy-default"]), + ], +) +async def test_default_metadata_tags_join_the_deployment_and_credential_tags( + make_core: MakeCore, host: ScriptedHost, request_metadata: Mapping[str, object], sent_tags: list[str] +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1", tags=["t1"], litellm_credential_name="cred")], + default_litellm_params=_PROXY_DEFAULT_TAGS, + ) + await core.acompletion(model="g", messages=MESSAGES, metadata=dict(request_metadata)) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["tags"] == sent_tags + + +async def test_default_metadata_tags_do_not_carry_one_request_tags_into_the_next( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params=_PROXY_DEFAULT_TAGS) + await core.acompletion(model="g", messages=MESSAGES, metadata={"tags": ["caller"]}) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[1].kwargs)["tags"] == ["proxy-default"] + + +@pytest.mark.parametrize( + ("request_metadata", "sent_guardrails"), + [ + ({"guardrails": []}, ["admin-guard"]), + ({"guardrails": ["caller-guard"]}, ["caller-guard", "admin-guard"]), + ({"guardrails": "caller-guard"}, ["admin-guard"]), + ({}, ["admin-guard"]), + ({"guardrails": [{"admin-guard": {"extra_body": {"threshold": 1}}}]}, ["admin-guard"]), + ( + { + "guardrails": [ + {"admin-guard": {"extra_body": {"threshold": 1}}, "caller-guard": {"extra_body": {"k": 2}}} + ] + }, + [{"caller-guard": {"extra_body": {"k": 2}}}, "admin-guard"], + ), + ], +) +async def test_a_default_guardrail_stays_on_with_its_own_settings_whatever_the_request_lists( + make_core: MakeCore, host: ScriptedHost, request_metadata: Mapping[str, object], sent_guardrails: list[object] +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"guardrails": ["admin-guard"]}} + ) + await core.acompletion(model="g", messages=MESSAGES, metadata=dict(request_metadata)) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["guardrails"] == sent_guardrails + + +async def test_a_default_guardrail_entry_keeps_the_admins_settings_over_the_callers( + make_core: MakeCore, host: ScriptedHost +) -> None: + admin_entry: Final = {"admin-guard": {"extra_body": {"threshold": 0.9}}} + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"guardrails": [admin_entry]}} + ) + await core.acompletion( # pyright: ignore[reportAttributeAccessIssue] # endpoint method + model="g", messages=MESSAGES, metadata={"guardrails": [{"admin-guard": {"extra_body": {"threshold": 0}}}]} + ) + assert _metadata(host.calls[0].kwargs)["guardrails"] == [admin_entry] + + +@pytest.mark.parametrize("router_caching_groups", [[("g", "other")], None]) +async def test_the_router_caching_groups_replace_the_request_caching_groups( + make_core: MakeCore, host: ScriptedHost, router_caching_groups: list[tuple[str, str]] | None +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], caching_groups=router_caching_groups) + await core.acompletion(model="g", messages=MESSAGES, metadata={"caching_groups": [("g", "premium")]}) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["caching_groups"] == router_caching_groups + + # --- sync completion runs the routing pre-call checks for a model group only ------------------- @@ -412,7 +663,7 @@ async def test_acompletion_counts_total_success_and_failed_calls_per_deployment_ make_core: MakeCore, host: ScriptedHost ) -> None: core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) - await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # endpoint method host.answer("openai/d1", rate_limit()) with pytest.raises(litellm.RateLimitError): await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue]