diff --git a/litellm/litellm_core_utils/routing_common_utils.py b/litellm/litellm_core_utils/routing_common_utils.py index d13ff4059aa..89d6729766f 100644 --- a/litellm/litellm_core_utils/routing_common_utils.py +++ b/litellm/litellm_core_utils/routing_common_utils.py @@ -1,6 +1,7 @@ import hashlib import json from collections.abc import Mapping, Sequence +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Final, TypeVar @@ -13,10 +14,12 @@ import litellm from litellm._logging import verbose_logger, verbose_router_logger from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure from litellm.llms.sagemaker.common_utils import deployment_inference_component +from litellm.secret_managers.main import str_to_bool from litellm.types.router import ( CredentialLiteLLMParams, Deployment, @@ -62,6 +65,74 @@ def without_router_only_kwargs(kwargs: Mapping[str, _V]) -> dict[str, _V]: return {key: value for key, value in kwargs.items() if key not in ROUTER_ONLY_CALL_KWARGS} +_ROUTER_OWNED_METADATA_KEYS: Final = frozenset({"caching_groups", "redis_namespace"}) +_PRIVATE_SETTING_BY_FLAG: Final = MappingProxyType({"mask_input": True, "mask_output": True, "trace_public": False}) +_JOINED_LIST_METADATA_KEYS: Final = frozenset({"tags", "guardrails"}) + + +def metadata_defaults_under_request( + defaults: Mapping[str, object], request_metadata: Mapping[str, object] +) -> Mapping[str, object]: + return { + key: _default_under_request(key=key, default=default, requested=request_metadata.get(key)) + for key, default in defaults.items() + } + + +def _default_under_request(key: str, default: object, requested: object) -> object: + if key in _ROUTER_OWNED_METADATA_KEYS or callable(default): + return default + if _PRIVATE_SETTING_BY_FLAG.get(key) is _is_flag_on(default): + return default + if key in _JOINED_LIST_METADATA_KEYS: + return _request_items_then_default_items(default=default, requested=requested) + return _request_value_over_default(default=default, requested=requested) + + +def _is_flag_on(value: object) -> bool: + parsed: Final = str_to_bool(value) if isinstance(value, str) else None + return bool(value) if parsed is None else parsed + + +def _request_items_then_default_items(default: object, requested: object) -> object: + default_items: Final = as_object_list(default) + if default_items is None: + return _request_value_over_default(default=default, requested=requested) + default_names: Final = frozenset(chain.from_iterable(_entry_names(item) for item in default_items)) + request_items: Final = tuple( + chain.from_iterable(_without_entries_named(item, default_names) for item in as_object_list(requested) or ()) + ) + return [*request_items, *(item for item in default_items if item not in request_items)] + + +def _entry_names(entry: object) -> frozenset[str]: + mapping: Final = as_str_mapping(entry) + if mapping is not None: + return frozenset(mapping) + return frozenset((entry,)) if isinstance(entry, str) else frozenset() + + +def _without_entries_named(entry: object, names: frozenset[str]) -> tuple[object, ...]: + mapping: Final = as_str_mapping(entry) + if mapping is None or names.isdisjoint(mapping): + return (entry,) + if names.issuperset(mapping): + return () + return ({key: value for key, value in mapping.items() if key not in names},) + + +def _request_value_over_default(default: object, requested: object) -> object: + if requested is None: + return default + default_mapping: Final = as_str_mapping(default) + if default_mapping is None: + return requested + requested_mapping: Final = as_str_mapping(requested) + if requested_mapping is None: + return default + return {**default_mapping, **requested_mapping} + + def as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]: if not isinstance(value, tuple): return () diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py index 7a5b52761e2..70c83bed728 100644 --- a/litellm/router_rust/invoke/endpoints.py +++ b/litellm/router_rust/invoke/endpoints.py @@ -2118,7 +2118,10 @@ class RustEndpoints: def text_completion(self, request: Mapping[str, object]) -> object: """The sync `text_completion`: one pick, no retries, the group name as the model.""" - from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin + from litellm.litellm_core_utils.routing_common_utils import ( + deployment_model_id_pin, + metadata_defaults_under_request, + ) kwargs: Final = without(request, "is_retry", "is_fallback", "is_async") model: Final = kwargs.get("model") @@ -2139,9 +2142,9 @@ class RustEndpoints: if key not in kwargs: kwargs[key] = value elif key == "metadata": - bucket = kwargs[key] - if isinstance(bucket, dict): - bucket.update(as_dict(value)) # pyright: ignore[reportUnknownMemberType] # the request's metadata dict + metadata_bucket(kwargs, key).update( + metadata_defaults_under_request(defaults=as_dict(value), request_metadata=as_dict(kwargs[key])) + ) pinned: Final = deployment_model_id_pin(data) if pinned is not None: kwargs["model_id"] = pinned diff --git a/litellm/router_rust/invoke/shaping.py b/litellm/router_rust/invoke/shaping.py index 258f919c9f1..bf6410d8242 100644 --- a/litellm/router_rust/invoke/shaping.py +++ b/litellm/router_rust/invoke/shaping.py @@ -32,7 +32,7 @@ from litellm.litellm_core_utils.core_helpers import without from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, inferred_provider from litellm.litellm_core_utils.request_timeout_resolver import get_service_tier_timeout -from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin +from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin, metadata_defaults_under_request from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy from litellm.responses.retrieve_stream_query import requests_streamed_retrieval, without_retrieve_stream_defaults from litellm.router_api import deployment_id as generate_deployment_id @@ -359,14 +359,15 @@ def update_kwargs_with_default_litellm_params( metadata_variable_name: str, ) -> None: """``_update_kwargs_with_default_litellm_params``: unset non-None defaults fill the request; - the default metadata is merged over the request's.""" + the default metadata fills the keys the request leaves unset, the router's caching groups excepted.""" defaults: Final = dict(without_retrieve_stream_defaults(shaping.default_litellm_params, kwargs)) metadata_defaults: Final = defaults.pop("metadata", {}) or {} # pyright: ignore[reportUnknownVariableType] # litellm's untyped helpers and request values for key, value in defaults.items(): if value is None: continue kwargs.setdefault(key, value) - metadata_bucket(kwargs, metadata_variable_name).update(as_dict(metadata_defaults)) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values + bucket: Final = metadata_bucket(kwargs, metadata_variable_name) + bucket.update(metadata_defaults_under_request(defaults=as_dict(metadata_defaults), request_metadata=bucket)) # pyright: ignore[reportUnknownArgumentType] # litellm's untyped helpers and request values # --- clientside credentials ------------------------------------------------------------------- diff --git a/tests/integration/providers/test_vertex_labels_router_default_metadata_wire.py b/tests/integration/providers/test_vertex_labels_router_default_metadata_wire.py new file mode 100644 index 00000000000..89747a5bed8 --- /dev/null +++ b/tests/integration/providers/test_vertex_labels_router_default_metadata_wire.py @@ -0,0 +1,85 @@ +import json +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +_MODEL_NAME: Final = "labelled-gemini" +_PROJECT: Final = "scripted-project" +_LOCATION: Final = "us-central1" +_ROUTER_LABELS: Final = {"env": "prod"} +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_REPLY: Final = json.dumps( + { + "candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 1, "totalTokenCount": 4}, + } +).encode() + + +def _gemini_peer(request: Request) -> Reply: + if request.target.endswith(":generateContent"): + return Reply(body=_REPLY) + return Reply(status=404, body=json.dumps({"error": request.target}).encode()) + + +def _config(directory: Path, gateway: Gateway, api_base: str) -> Path: + config: Final = directory / "router-default-requester-metadata.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": _MODEL_NAME, + "litellm_params": { + "model": "vertex_ai/gemini-3.8-flash", + "api_base": api_base, + "vertex_project": _PROJECT, + "vertex_location": _LOCATION, + "vertex_credentials": service_account_json(_PROJECT, gateway.upstream_url.rstrip("/")), + }, + } + ], + "router_settings": {"default_litellm_params": {"metadata": {"requester_metadata": _ROUTER_LABELS}}}, + } + ), + encoding="utf-8", + ) + return config + + +def _labels_sent(proxy: Gateway, wire: Wire, metadata: Mapping[str, JsonValue] | None) -> JsonValue: + response: Final = proxy.request( + "POST", + "/v1/chat/completions", + { + "model": _MODEL_NAME, + "messages": [{"role": "user", "content": "Return exactly OK."}], + **({"metadata": dict(metadata)} if metadata is not None else {}), + }, + ) + assert response.status_code == 200, response.text + generations: Final = tuple(request for request in wire.drain() if request.target.endswith(":generateContent")) + assert len(generations) == 1, generations + return _JSON_OBJECT.validate_json(generations[0].body).get("labels") + + +def test_router_default_requester_metadata_merges_under_each_callers_vertex_labels( + gateway: Gateway, tmp_path: Path +) -> None: + with ( + wire_server(_gemini_peer) as wire, + owned_proxy_process(gateway, tmp_path, {}, config=_config(tmp_path, gateway, wire.url)) as owned, + ): + proxy: Final = Gateway(owned.gateway.client, owned.gateway.key, gateway.upstream_url) + assert _labels_sent(proxy, wire, {"team": "ai-platform"}) == {**_ROUTER_LABELS, "team": "ai-platform"} + assert _labels_sent(proxy, wire, {"env": "staging"}) == {"env": "staging"} + assert _labels_sent(proxy, wire, None) == _ROUTER_LABELS diff --git a/tests/unit/test_router/core/test_endpoints_characterization_inference.py b/tests/unit/test_router/core/test_endpoints_characterization_inference.py index 8db0d6bea12..c975e45b1d3 100644 --- a/tests/unit/test_router/core/test_endpoints_characterization_inference.py +++ b/tests/unit/test_router/core/test_endpoints_characterization_inference.py @@ -205,6 +205,17 @@ async def test_sync_text_completion_merges_default_metadata_into_request_metadat assert (metadata["x"], metadata["y"], metadata["model_group"], sent["user"]) == (1, 2, "g", "u") +async def test_sync_text_completion_keeps_the_request_metadata_over_the_default_metadata( + make_core: MakeCore, host: EndpointHost +): + core: Final = make_core( + model_list=[deployment("g", "d1")], + default_litellm_params={"metadata": {"requester_metadata": {"env": "prod", "region": "us"}}}, + ) + core.text_completion(model="g", prompt="p", metadata={"requester_metadata": {"env": "dev", "team": "t1"}}) + assert _bucket(_sent(host))["requester_metadata"] == {"env": "dev", "region": "us", "team": "t1"} + + async def test_sync_text_completion_sends_the_model_group_name_not_the_deployment_model( make_core: MakeCore, host: EndpointHost ): @@ -329,12 +340,12 @@ async def test_default_litellm_params_metadata_merges_into_request_metadata(make assert (metadata["team"], metadata["trace"], metadata["model_group"]) == ("t", "x", "g") -async def test_default_litellm_params_metadata_overwrites_the_same_request_metadata_key( +async def test_the_request_metadata_key_wins_over_the_same_default_litellm_params_metadata_key( make_core: MakeCore, host: EndpointHost ): core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"team": "t"}}) await core.acompletion(model="g", messages=MESSAGES, metadata={"team": "request"}) - assert _bucket(_sent(host))["team"] == "t" + assert _bucket(_sent(host))["team"] == "request" async def test_default_litellm_params_metadata_goes_to_litellm_metadata_for_generic_endpoints( diff --git a/tests/unit/test_router/core/test_gaps_l2800.py b/tests/unit/test_router/core/test_gaps_l2800.py index 78e237f0696..196ce20d4b3 100644 --- a/tests/unit/test_router/core/test_gaps_l2800.py +++ b/tests/unit/test_router/core/test_gaps_l2800.py @@ -246,7 +246,7 @@ async def test_responses_replace_kubernetes_pod_routing_in_the_request_metadata( """On the Responses path the routing goes to ``litellm_metadata``; a pod routing key the caller put in ``metadata`` is overwritten with the serving deployment's (none here).""" core: Final = make_core(model_list=[deployment("g", "d1")]) - await core.aresponses( # pyright: ignore[reportAttributeAccessIssue] + await core.aresponses( # pyright: ignore[reportAttributeAccessIssue] # endpoint method model="g", input="hi", metadata={KUBERNETES_POD_ROUTING_KEY: {"pod": "stale"}, "keep": 1} ) sent: Final = host.calls[0].kwargs @@ -332,7 +332,7 @@ async def test_the_process_request_timeout_outranks_the_router_timeout_once_a_ro process settings), and only when the router has a ``timeout`` of its own.""" monkeypatch.setattr(litellm, "request_timeout", 33.0) core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42) - await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # endpoint method assert host.calls[0].kwargs["timeout"] == 33.0 @@ -353,7 +353,7 @@ async def test_default_litellm_params_timeout_applies_when_no_router_or_process_ ) -> None: host.set_dynamic(request_timeout=None) core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params={"timeout": 9}) - await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # endpoint method assert host.calls[0].kwargs["timeout"] == 9 @@ -368,6 +368,257 @@ async def test_default_litellm_params_stream_timeout_is_the_last_stream_timeout_ assert host.calls[0].kwargs["stream_timeout"] == 4 +_PROXY_REQUESTER_METADATA: Final = {"metadata": {"requester_metadata": {"env": "prod"}}} + + +@pytest.mark.parametrize( + ("request_metadata", "sent_requester_metadata"), + [ + ({"requester_metadata": {"team": "t1"}}, {"env": "prod", "team": "t1"}), + ({"requester_metadata": {"env": "dev", "team": "t1"}}, {"env": "dev", "team": "t1"}), + ({"requester_metadata": None}, {"env": "prod"}), + ({"requester_metadata": {}}, {"env": "prod"}), + ({"requester_metadata": "x"}, {"env": "prod"}), + ({}, {"env": "prod"}), + ], +) +async def test_default_requester_metadata_merges_under_the_request_requester_metadata( + make_core: MakeCore, + host: ScriptedHost, + request_metadata: Mapping[str, object], + sent_requester_metadata: Mapping[str, str], +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params=_PROXY_REQUESTER_METADATA) + await core.acompletion(model="g", messages=MESSAGES, metadata=dict(request_metadata)) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["requester_metadata"] == sent_requester_metadata + + +async def test_responses_default_requester_metadata_merges_under_the_request_litellm_metadata( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params=_PROXY_REQUESTER_METADATA) + await core.aresponses( # pyright: ignore[reportAttributeAccessIssue] # endpoint method + model="g", input="hi", litellm_metadata={"requester_metadata": {"team": "t1"}} + ) + sent: Final = _metadata(host.calls[0].kwargs, "litellm_metadata") + assert sent["requester_metadata"] == {"env": "prod", "team": "t1"} + + +async def test_default_requester_metadata_does_not_carry_one_request_labels_into_the_next( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params=_PROXY_REQUESTER_METADATA) + await core.acompletion(model="g", messages=MESSAGES, metadata={"requester_metadata": {"team": "t1"}}) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[1].kwargs)["requester_metadata"] == {"env": "prod"} + + +@pytest.mark.parametrize( + ("request_metadata", "sent_regions"), + [({"regions": ["eu"]}, ["eu"]), ({"regions": None}, ["us"]), ({}, ["us"])], +) +async def test_a_default_metadata_list_applies_only_when_the_request_sets_none( + make_core: MakeCore, host: ScriptedHost, request_metadata: Mapping[str, object], sent_regions: list[str] +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"regions": ["us"]}} + ) + await core.acompletion(model="g", messages=MESSAGES, metadata=dict(request_metadata)) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["regions"] == sent_regions + + +@pytest.mark.parametrize("mask_key", ["mask_input", "mask_output"]) +@pytest.mark.parametrize( + ("default_mask", "request_mask", "sent_mask"), + [ + (True, False, True), + (True, None, True), + ("true", False, "true"), + (False, True, True), + (False, None, False), + ("false", True, True), + ], +) +async def test_default_metadata_masking_stays_on_once_either_side_turns_it_on( + make_core: MakeCore, + host: ScriptedHost, + mask_key: str, + default_mask: object, + request_mask: object, + sent_mask: object, +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {mask_key: default_mask}} + ) + await core.acompletion(model="g", messages=MESSAGES, metadata={mask_key: request_mask}) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)[mask_key] == sent_mask + + +@pytest.mark.parametrize( + ("default_public", "request_public", "sent_public"), + [ + (False, True, False), + ("false", True, "false"), + (False, None, False), + (True, False, False), + (True, None, True), + ("true", True, True), + ], +) +async def test_a_private_default_trace_stays_private_whatever_the_request_sends( + make_core: MakeCore, + host: ScriptedHost, + default_public: object, + request_public: object, + sent_public: object, +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"trace_public": default_public}} + ) + await core.acompletion(model="g", messages=MESSAGES, metadata={"trace_public": request_public}) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["trace_public"] == sent_public + + +async def test_a_default_redis_namespace_beats_the_request_redis_namespace( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"redis_namespace": "tenant-a"}} + ) + await core.acompletion(model="g", messages=MESSAGES, metadata={"redis_namespace": "tenant-b"}) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["redis_namespace"] == "tenant-a" + + +def _redact(_value: object) -> str: + return "redacted" + + +@pytest.mark.parametrize("request_value", ["", {"caller": "value"}]) +async def test_a_default_masking_function_beats_whatever_the_request_sends( + make_core: MakeCore, host: ScriptedHost, request_value: object +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"langfuse_masking_function": _redact}} + ) + await core.acompletion(model="g", messages=MESSAGES, metadata={"langfuse_masking_function": request_value}) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["langfuse_masking_function"] is _redact + + +_PROXY_DEFAULT_TAGS: Final = {"metadata": {"tags": ["proxy-default"]}} + + +@pytest.mark.parametrize( + ("request_metadata", "sent_tags"), + [ + ({"tags": ["caller"]}, ["caller", "proxy-default"]), + ({"tags": ["proxy-default", "caller"]}, ["proxy-default", "caller"]), + ({"tags": None}, ["proxy-default"]), + ({}, ["proxy-default"]), + ({"tags": "caller"}, ["proxy-default"]), + ], +) +async def test_default_metadata_tags_follow_the_request_tags( + make_core: MakeCore, host: ScriptedHost, request_metadata: Mapping[str, object], sent_tags: object +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params=_PROXY_DEFAULT_TAGS) + await core.acompletion(model="g", messages=MESSAGES, metadata=dict(request_metadata)) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["tags"] == sent_tags + + +@pytest.mark.parametrize( + ("request_metadata", "sent_tags"), + [ + ({"tags": ["caller"]}, ["caller"]), + ({}, "proxy-default"), + ], +) +async def test_a_default_tags_that_is_not_a_list_yields_to_the_request_tags( + make_core: MakeCore, host: ScriptedHost, request_metadata: Mapping[str, object], sent_tags: object +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"tags": "proxy-default"}} + ) + await core.acompletion(model="g", messages=MESSAGES, metadata=dict(request_metadata)) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["tags"] == sent_tags + + +@pytest.mark.parametrize( + ("request_metadata", "sent_tags"), + [ + ({}, ["t1", "Credential: cred", "proxy-default"]), + ({"tags": ["caller"]}, ["caller", "t1", "Credential: cred", "proxy-default"]), + ], +) +async def test_default_metadata_tags_join_the_deployment_and_credential_tags( + make_core: MakeCore, host: ScriptedHost, request_metadata: Mapping[str, object], sent_tags: list[str] +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1", tags=["t1"], litellm_credential_name="cred")], + default_litellm_params=_PROXY_DEFAULT_TAGS, + ) + await core.acompletion(model="g", messages=MESSAGES, metadata=dict(request_metadata)) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["tags"] == sent_tags + + +async def test_default_metadata_tags_do_not_carry_one_request_tags_into_the_next( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], default_litellm_params=_PROXY_DEFAULT_TAGS) + await core.acompletion(model="g", messages=MESSAGES, metadata={"tags": ["caller"]}) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[1].kwargs)["tags"] == ["proxy-default"] + + +@pytest.mark.parametrize( + ("request_metadata", "sent_guardrails"), + [ + ({"guardrails": []}, ["admin-guard"]), + ({"guardrails": ["caller-guard"]}, ["caller-guard", "admin-guard"]), + ({"guardrails": "caller-guard"}, ["admin-guard"]), + ({}, ["admin-guard"]), + ({"guardrails": [{"admin-guard": {"extra_body": {"threshold": 1}}}]}, ["admin-guard"]), + ( + { + "guardrails": [ + {"admin-guard": {"extra_body": {"threshold": 1}}, "caller-guard": {"extra_body": {"k": 2}}} + ] + }, + [{"caller-guard": {"extra_body": {"k": 2}}}, "admin-guard"], + ), + ], +) +async def test_a_default_guardrail_stays_on_with_its_own_settings_whatever_the_request_lists( + make_core: MakeCore, host: ScriptedHost, request_metadata: Mapping[str, object], sent_guardrails: list[object] +) -> None: + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"guardrails": ["admin-guard"]}} + ) + await core.acompletion(model="g", messages=MESSAGES, metadata=dict(request_metadata)) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["guardrails"] == sent_guardrails + + +async def test_a_default_guardrail_entry_keeps_the_admins_settings_over_the_callers( + make_core: MakeCore, host: ScriptedHost +) -> None: + admin_entry: Final = {"admin-guard": {"extra_body": {"threshold": 0.9}}} + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"metadata": {"guardrails": [admin_entry]}} + ) + await core.acompletion( # pyright: ignore[reportAttributeAccessIssue] # endpoint method + model="g", messages=MESSAGES, metadata={"guardrails": [{"admin-guard": {"extra_body": {"threshold": 0}}}]} + ) + assert _metadata(host.calls[0].kwargs)["guardrails"] == [admin_entry] + + +@pytest.mark.parametrize("router_caching_groups", [[("g", "other")], None]) +async def test_the_router_caching_groups_replace_the_request_caching_groups( + make_core: MakeCore, host: ScriptedHost, router_caching_groups: list[tuple[str, str]] | None +) -> None: + core: Final = make_core(model_list=[deployment("g", "d1")], caching_groups=router_caching_groups) + await core.acompletion(model="g", messages=MESSAGES, metadata={"caching_groups": [("g", "premium")]}) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + assert _metadata(host.calls[0].kwargs)["caching_groups"] == router_caching_groups + + # --- sync completion runs the routing pre-call checks for a model group only ------------------- @@ -412,7 +663,7 @@ async def test_acompletion_counts_total_success_and_failed_calls_per_deployment_ make_core: MakeCore, host: ScriptedHost ) -> None: core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) - await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # endpoint method host.answer("openai/d1", rate_limit()) with pytest.raises(litellm.RateLimitError): await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue]