From e5aa3d32cde109d236881ff5473d3d43ff0a384e Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:16:42 -0700 Subject: [PATCH] fix(proxy): forward a valid metadata.user_id to Anthropic Messages on pass-through routes (#49333) * fix(proxy): forward the client's own metadata on Anthropic and custom pass-through routes * fix(proxy): forward only a valid metadata.user_id to Anthropic Messages on pass-through routes The whole-object forward sent keys Anthropic refuses and reached other providers, so a request that used to pass could start failing with a 400. Read the client's user_id after hooks, drop it unless it is a string with no email or phone number, and send only that key when the upstream URL path ends in /v1/messages. Share the validator with the translated route, whose email check now matches any embedded address as Anthropic's does. * fix(proxy): forward metadata.user_id only to Anthropic upstreams no guardrail scans Limit the pass-through user_id to Messages create on api.anthropic.com or the configured Anthropic base, mirror the checks Anthropic answers 400 for, and keep the field off any request a pre-call guardrail inspects. The translated route goes back to its own user_id filter. * fix(proxy): judge user_id forwarding on the whole request and cap it for configured Anthropic bases * refactor(proxy): pass the user_id check a plain mapping instead of an optional body * fix(proxy): drop an unprintable metadata.user_id and warn when a refused id is dropped Anthropic's whitespace is not Python's, so a control character before the at sign hid an email from the filter and turned a 200 into a 400. Fireworks and Moonshot also fail on newline ids. Requiring a printable id covers both, and a refused id now logs a warning that never carries the value. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/constants.py | 2 + litellm/llms/anthropic/common_utils.py | 37 +++ .../llm_passthrough_endpoints.py | 8 +- .../pass_through_endpoints.py | 54 +++- litellm/proxy/utils.py | 13 +- .../test_anthropic_passthrough_wire.py | 1 - .../anthropic/test_anthropic_common_utils.py | 92 ++++++ .../test_pass_through_endpoints.py | 268 +++++++++++++++++- .../utils/proxy_logging/test_pre_call_hook.py | 18 ++ 9 files changed, 479 insertions(+), 14 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index c17e776f54b..28e7b4a7fd7 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -634,6 +634,8 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024) # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 +ANTHROPIC_MAX_USER_ID_LENGTH: Final = 512 +ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH: Final = 255 ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:" REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5) DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index e1f9a7d540f..e0da8943a4a 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -4,6 +4,7 @@ This file contains common utils for anthropic calls. import copy import json +import os import re from collections.abc import Container, Mapping, MutableMapping, Sequence from dataclasses import dataclass @@ -19,6 +20,8 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import ( + ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, + ANTHROPIC_MAX_USER_ID_LENGTH, DEFAULT_MODEL_CREATED_AT_TIME, DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, @@ -178,6 +181,40 @@ def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: boo return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools] +def anthropic_pass_through_api_base() -> str: + return os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com" + + +def _same_host_and_port(url: httpx.URL, other: str) -> bool: + try: + other_url: Final = httpx.URL(other) + except httpx.InvalidURL: + return False + return (url.host, url.port) == (other_url.host, other_url.port) + + +def anthropic_upstream_user_id_limit(url: str) -> int | None: + upstream: Final = httpx.URL(url) + if not upstream.path.removesuffix("/").endswith("/v1/messages"): + return None + if upstream.host == "api.anthropic.com": + return ANTHROPIC_MAX_USER_ID_LENGTH + return ( + ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH + if _same_host_and_port(upstream, anthropic_pass_through_api_base()) + else None + ) + + +def _has_email_shape(token: str) -> bool: + at_sign: Final = token.find("@", 1) + return at_sign != -1 and token.find(".", at_sign + 2, -1) != -1 + + +def anthropic_accepts_user_id(user_id: str, max_length: int) -> bool: + return len(user_id) <= max_length and user_id.isprintable() and not any(map(_has_email_shape, user_id.split())) + + def is_anthropic_messages_url(url: str) -> bool: """Check whether a URL addresses Anthropic's Messages API.""" parsed_url: Final = urlparse(url) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 33b18c3d939..151938d73d7 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -52,7 +52,11 @@ from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization -from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers +from litellm.llms.anthropic.common_utils import ( + AnthropicModelInfo, + anthropic_pass_through_api_base, + merge_anthropic_beta_headers, +) from litellm.llms.azure.passthrough.transformation import ( foreign_azure_deployment, is_azure_body_model_inference_endpoint, @@ -1047,7 +1051,7 @@ async def anthropic_proxy_route( """ [Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion) """ - base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com" + base_target_url = anthropic_pass_through_api_base() encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 647e145aac1..bda4f4c2ad2 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -71,7 +71,11 @@ from litellm.litellm_core_utils.litellm_logging import get_masked_values from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.redact_messages import should_redact_message_logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps -from litellm.llms.anthropic.common_utils import is_anthropic_messages_url +from litellm.llms.anthropic.common_utils import ( + anthropic_accepts_user_id, + anthropic_upstream_user_id_limit, + is_anthropic_messages_url, +) from litellm.llms.base_llm.managed_resources.utils import ( resolve_passthrough_managed_id_provider, ) @@ -503,7 +507,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): url: httpx.URL, headers: dict, requested_query_params: dict | None = None, - _parsed_body: dict | None = None, + _parsed_body: Mapping[str, object] | None = None, forward_multipart: bool = False, ) -> httpx.Response: """ @@ -717,8 +721,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): # unenforced. Set after the client merge so a request body cannot supply its own budget. # # Only for the built-in provider routes. `get_model_from_request` returns - # None for a user-defined pass-through, deliberately: its body is forwarded - # verbatim, so `model` there names an UPSTREAM model rather than a + # None for a user-defined pass-through, deliberately: the client's `model` + # goes upstream unchanged, so it names an UPSTREAM model rather than a # LiteLLM-managed one. Enforcement is therefore skipped on those routes, and # charging a counter anyway would track spend that nothing can refuse, and # would attribute it to a budget the operator scoped to a LiteLLM model that @@ -1187,6 +1191,36 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]: return headers +def _json_object(value: object) -> Mapping[str, object]: + return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({}) # cast-ok: json + + +def _anthropic_user_id_to_forward( + body: Mapping[str, object], + url: str, + has_pre_call_guardrails: Callable[[Mapping[str, object]], bool], +) -> str | None: + user_id: Final = _json_object(body.get("metadata")).get("user_id") + if not isinstance(user_id, str): + return None + max_length: Final = anthropic_upstream_user_id_limit(url) + if max_length is None: + return None + if not anthropic_accepts_user_id(user_id, max_length): + verbose_proxy_logger.warning("Not forwarding metadata.user_id because the upstream would refuse it") + return None + if has_pre_call_guardrails(body): + verbose_proxy_logger.debug("Not forwarding metadata.user_id because a pre-call guardrail cannot scan it") + return None + return user_id + + +def _with_anthropic_user_id(body: Mapping[str, object] | None, user_id: str | None) -> Mapping[str, object] | None: + if body is None or user_id is None: + return body + return {**body, "metadata": {"user_id": user_id}} + + def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None: return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None @@ -1423,6 +1457,10 @@ async def pass_through_request( request_method=getattr(request, "method", None), cost_per_request=cost_per_request, ) + hooked_body: Final = _json_object(_parsed_body) + client_user_id: Final = _anthropic_user_id_to_forward( + hooked_body, str(url), proxy_logging_obj.request_has_pre_call_guardrails + ) kwargs = HttpPassThroughEndpointHelpers.init_kwargs_for_pass_through_endpoint( # rebind-ok: pre-existing rebinding on a rename-only line user_api_key_dict=user_api_key_dict, _parsed_body=_parsed_body, @@ -1590,11 +1628,13 @@ async def pass_through_request( else: logging_url = str(url) + "?" + requested_query_params_str + outgoing_body: Final = cast("Mapping[str, object] | None", _parsed_body) # cast-ok: json + upstream_body: Final = _with_anthropic_user_id(outgoing_body, client_user_id) logging_obj.pre_call( input=[{"role": "user", "content": safe_dumps(_parsed_body)}], api_key="", additional_args={ - "complete_input_dict": _parsed_body, + "complete_input_dict": upstream_body, "api_base": str(logging_url), "headers": redact_credential_headers(upstream_headers), }, @@ -1634,7 +1674,7 @@ async def pass_through_request( url, params=requested_query_params, headers=upstream_headers, - json=_parsed_body, + json=upstream_body, ) ) @@ -1733,7 +1773,7 @@ async def pass_through_request( url=url, headers=upstream_headers, requested_query_params=requested_query_params, - _parsed_body=_parsed_body, + _parsed_body=upstream_body, forward_multipart=is_multipart, ) verbose_proxy_logger.debug("response.headers= %s", response.headers) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9e237ffcb57..c7efbf9dacf 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3031,14 +3031,21 @@ class ProxyLogging: hook to consult, but it judges the payload the same way, so a proxy configured only with one of those still has something to say about every record. """ - if request_metadata.get("_guardrail_pipelines"): + return self.request_has_pre_call_guardrails({"metadata": dict(request_metadata)}, event_type) + + def request_has_pre_call_guardrails( + self, + request_data: Mapping[str, object], + event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call, + ) -> bool: + if _policy_state_metadata(request_data).get("_guardrail_pipelines"): return True caps: Final = ProxyLogging.callback_capabilities() if caps.has_content_enforcer: return True - probe: Final = {"metadata": dict(request_metadata)} return any( - isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type) + isinstance(callback, CustomGuardrail) + and callback.should_run_guardrail(data=request_data, event_type=event_type) for callback in caps.resolved_callbacks ) diff --git a/tests/integration/providers/test_anthropic_passthrough_wire.py b/tests/integration/providers/test_anthropic_passthrough_wire.py index a791eb9c2a2..0a16520f626 100644 --- a/tests/integration/providers/test_anthropic_passthrough_wire.py +++ b/tests/integration/providers/test_anthropic_passthrough_wire.py @@ -317,7 +317,6 @@ def test_credentialless_proxy_forwards_the_caller_anthropic_credential_and_never def test_anthropic_sdk_metadata_user_id_reaches_upstream(api_key_proxy: Gateway, wire: Wire) -> None: - pytest.skip("BUG: /anthropic/v1/messages drops the native metadata.user_id field before forwarding upstream") with api_key_proxy.scenario() as scenario: key: Final = scenario.key() sent: Final[list[httpx.Request]] = [] diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index d18ef28c68e..cf0b8a39865 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -56,6 +56,98 @@ def test_is_anthropic_messages_url(url: str, expected: bool) -> None: assert is_anthropic_messages_url(url) is expected +@pytest.mark.parametrize( + ("configured_base", "url", "expected"), + [ + pytest.param(None, "https://api.anthropic.com/v1/messages", 512, id="default-upstream"), + pytest.param(None, "https://api.anthropic.com:443/v1/messages/", 512, id="default-upstream-explicit-port"), + pytest.param("http://relay.test:8080", "https://api.anthropic.com/v1/messages", 512, id="direct-target"), + pytest.param("https://api.anthropic.com", "https://api.anthropic.com/v1/messages", 512, id="configured-default"), + pytest.param("http://relay.test:8080", "http://relay.test:8080/v1/messages", 255, id="configured-upstream"), + pytest.param("http://relay.test:8080/a", "http://relay.test:8080/a/v1/messages", 255, id="configured-prefix"), + pytest.param("https://Relay.Test", "https://relay.test/v1/messages", 255, id="configured-upper-case-host"), + pytest.param("https://relay.test:443", "https://relay.test/v1/messages", 255, id="configured-https-port"), + pytest.param("http://relay.test:80/a", "http://relay.test/a/v1/messages", 255, id="configured-http-port"), + pytest.param("http://relay.test:8080", "http://relay.test:8081/v1/messages", None, id="other-port"), + pytest.param("http://relay.test:notaport", "http://relay.test/v1/messages", None, id="invalid-configured-base"), + pytest.param(None, "https://other-vendor.test/v1/messages", None, id="other-vendor"), + pytest.param(None, "https://api.anthropic.com/v1/messages/count_tokens", None, id="count-tokens"), + ], +) +def test_anthropic_upstream_user_id_limit( + monkeypatch: pytest.MonkeyPatch, configured_base: str | None, url: str, expected: int | None +) -> None: + from litellm.llms.anthropic.common_utils import anthropic_upstream_user_id_limit + + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) + if configured_base is not None: + monkeypatch.setenv("ANTHROPIC_API_BASE", configured_base) + + assert anthropic_upstream_user_id_limit(url) == expected + + +def test_anthropic_pass_through_api_base_prefers_the_api_base_variable(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.anthropic.common_utils import anthropic_pass_through_api_base + + monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://fallback.test") + monkeypatch.setenv("ANTHROPIC_API_BASE", "http://preferred.test") + assert anthropic_pass_through_api_base() == "http://preferred.test" + + monkeypatch.delenv("ANTHROPIC_API_BASE") + assert anthropic_pass_through_api_base() == "http://fallback.test" + + +# Each row was sent to https://api.anthropic.com/v1/messages on 2026-10-30, False rows answered 400. +@pytest.mark.parametrize( + ("user_id", "expected"), + [ + pytest.param("user-1", True, id="opaque"), + pytest.param("", True, id="empty"), + pytest.param('{"device_id":"abc","session_id":"def"}', True, id="claude-code-shape"), + pytest.param("1234567890", True, id="digits"), + pytest.param("+14155550100", True, id="phone-shape"), + pytest.param("a@b", True, id="no-dot"), + pytest.param("@b.co", True, id="nothing-before-at-sign"), + pytest.param("a@.co", True, id="nothing-between-at-sign-and-dot"), + pytest.param("a@b.", True, id="nothing-after-dot"), + pytest.param("a @ b.co", True, id="spaces-around-at-sign"), + pytest.param("a\u00a0@b.co", False, id="no-break-space-before-at-sign"), + pytest.param("a@b\n.co", False, id="newline-before-dot"), + pytest.param("line one\nline two", False, id="newline"), + pytest.param("a\tb", False, id="tab"), + pytest.param("a\x1f@b.co", False, id="unit-separator-before-at-sign"), + pytest.param("a@\x1cb.co", False, id="file-separator-after-at-sign"), + pytest.param("a\u200bb", False, id="zero-width-space"), + pytest.param("a@b.co", False, id="email"), + pytest.param("user a@b.co here", False, id="email-inside-text"), + pytest.param('{"device_id":"a@b.co"}', False, id="email-inside-json"), + pytest.param("a@@example.com", False, id="doubled-at-sign"), + pytest.param("@@example.com", False, id="leading-doubled-at-sign"), + pytest.param("a@b@c.co", False, id="two-at-signs"), + pytest.param("a@b..co", False, id="doubled-dot"), + pytest.param("a@.b.co", False, id="dot-after-at-sign"), + pytest.param("a@b.c d", False, id="email-before-space"), + ], +) +def test_anthropic_accepts_user_id(user_id: str, expected: bool) -> None: + from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH + from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id + + assert anthropic_accepts_user_id(user_id, ANTHROPIC_MAX_USER_ID_LENGTH) is expected + + +@pytest.mark.parametrize("character", ["a", "\u00e9", "\U0001f600"], ids=["ascii", "two-bytes", "four-bytes"]) +def test_anthropic_accepts_user_id_counts_characters_up_to_the_limit(character: str) -> None: + from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH + from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id + + assert anthropic_accepts_user_id(character * ANTHROPIC_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH) is True + assert ( + anthropic_accepts_user_id(character * (ANTHROPIC_MAX_USER_ID_LENGTH + 1), ANTHROPIC_MAX_USER_ID_LENGTH) is False + ) + + def test_anthropic_model_not_found_maps_to_not_found_but_authentication_error_does_not() -> None: with respx.mock(assert_all_called=True) as router: router.post("https://api.anthropic.com/v1/messages").mock( diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index b7e76ce0068..4fe358c38a3 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -31,7 +31,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile import litellm from litellm._logging import verbose_proxy_logger from litellm.caching.dual_cache import DualCache -from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS +from litellm.constants import ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH, DEFAULT_REQUEST_TIMEOUT_SECONDS from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.prometheus import PrometheusLogger @@ -6621,6 +6621,272 @@ async def test_pass_through_body_stays_byte_identical_outside_the_opt_in( assert "x-litellm-call-id" in response.headers +def _echo_upstream_body(upstream_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, content=upstream_request.content, headers={"content-type": "application/json"}, request=upstream_request + ) + + +async def _relayed_body(response: Response) -> bytes: + if isinstance(response, StreamingResponse): + return b"".join([chunk async for chunk in response.body_iterator]) + return bytes(response.body) + + +async def _relay_through_pass_through( + client_body: Mapping[str, object], + target: str, + guardrails_config: dict[str, object] | None, + custom_body: dict[str, object] | None = None, +) -> Mapping[str, object]: + async def receive() -> Mapping[str, object]: + return {"type": "http.request", "body": json.dumps(dict(client_body)).encode()} + + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": "/my-claude-route", + "query_string": b"", + "headers": [(b"content-type", b"application/json")], + }, + receive, + ) + fake_client, cleanup = _inject_fake_passthrough_client(httpx.MockTransport(_echo_upstream_body), timeout=None) + try: + with ExitStack() as stack: + _enter_relay_logging_mocks(stack, {}) + stack.enter_context( + patch("litellm.proxy.proxy_server.proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache())) + ) + response: Final = await pass_through_request( + request=request, + target=target, + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"), + custom_body=custom_body, + guardrails_config=guardrails_config, + ) + relayed: Final = await _relayed_body(response) + finally: + cleanup() + await fake_client.aclose() + + assert response.status_code == 200 + return MappingProxyType(json.loads(relayed)) + + +_MESSAGES_BODY: Final = MappingProxyType( + {"model": "claude-sonnet-4-5", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]} +) +_ANTHROPIC_MESSAGES: Final = "https://api.anthropic.com/v1/messages" + + +@pytest.fixture +def no_configured_anthropic_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +@pytest.mark.parametrize( + "client_metadata, target, forwarded_metadata", + [ + ({"user_id": "user-1"}, _ANTHROPIC_MESSAGES, {"user_id": "user-1"}), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/", {"user_id": "user-1"}), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}?beta=true", {"user_id": "user-1"}), + ( + {"user_id": "user-1", "tags": ["a"], "trace_id": "t", "generation_name": "g", "foo": "bar"}, + _ANTHROPIC_MESSAGES, + {"user_id": "user-1"}, + ), + ({"user_id": "1234567890"}, _ANTHROPIC_MESSAGES, {"user_id": "1234567890"}), + ({"tags": ["a"], "foo": "bar"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "a@example.com"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "x" * (ANTHROPIC_MAX_USER_ID_LENGTH + 1)}, _ANTHROPIC_MESSAGES, None), + ({"user_id": 42}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "line one\nline two"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "a\x1f@example.com"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/count_tokens", None), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/batches", None), + ({"user_id": "user-1"}, "https://api.anthropic.com/v1/models", None), + ({"user_id": "user-1"}, "https://other-vendor.test/anthropic/v1/messages", None), + ({"user_id": "user-1"}, "https://other-vendor.test/v1/responses", None), + ({"user_id": "user-1"}, "https://other-vendor.test/v1/chat/completions", None), + ], + ids=[ + "messages", + "messages-trailing-slash", + "messages-with-query", + "user-id-beside-litellm-and-unknown-keys", + "digits-user-id", + "no-user-id", + "email-user-id", + "over-long-user-id", + "integer-user-id", + "newline-user-id", + "control-character-before-at-sign", + "count-tokens", + "batches", + "other-anthropic-path", + "other-vendor-messages", + "other-vendor-responses", + "other-vendor-chat-completions", + ], +) +async def test_pass_through_forwards_only_an_acceptable_user_id_to_anthropic_messages( + stream: bool, client_metadata: object, target: str, forwarded_metadata: Mapping[str, object] | None +) -> None: + client_body: Final = MappingProxyType( + {**_MESSAGES_BODY, "metadata": client_metadata, **({"stream": True} if stream else {})} + ) + body_without_metadata: Final = {k: v for k, v in client_body.items() if k != "metadata"} + + relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None) + + assert relayed == ( + body_without_metadata if forwarded_metadata is None else {**body_without_metadata, "metadata": forwarded_metadata} + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize("base_env", ["ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"]) +@pytest.mark.parametrize( + "target, forwarded", + [ + ("http://gateway.test:8080/anthropic/v1/messages", True), + ("http://gateway.test:9090/v1/messages", False), + ("http://other-gateway.test:8080/v1/messages", False), + ], + ids=["configured-upstream", "other-port", "other-host"], +) +async def test_pass_through_treats_the_configured_anthropic_base_as_anthropic( + monkeypatch: pytest.MonkeyPatch, base_env: str, target: str, forwarded: bool +) -> None: + monkeypatch.setenv(base_env, "http://gateway.test:8080/anthropic") + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}}) + + relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None) + + assert relayed == (client_body if forwarded else _MESSAGES_BODY) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize( + "user_id_length, target, forwarded", + [ + (ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, "http://gateway.test/v1/messages", True), + (ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH + 1, "http://gateway.test/v1/messages", False), + (ANTHROPIC_MAX_USER_ID_LENGTH, _ANTHROPIC_MESSAGES, True), + ], + ids=["at-the-configured-base-limit", "over-the-configured-base-limit", "direct-anthropic-keeps-its-limit"], +) +async def test_pass_through_caps_the_user_id_for_a_configured_anthropic_compatible_base( + monkeypatch: pytest.MonkeyPatch, user_id_length: int, target: str, forwarded: bool +) -> None: + monkeypatch.setenv("ANTHROPIC_API_BASE", "http://gateway.test") + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "u" * user_id_length}}) + + relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None) + + assert relayed == (client_body if forwarded else _MESSAGES_BODY) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize( + "default_on, event_hook, guardrails_config, client_fields, stream_scope, forwarded", + [ + (False, GuardrailEventHooks.pre_call, None, {}, None, True), + (False, GuardrailEventHooks.pre_call, {"unregistered-guard": {}}, {}, None, True), + (True, GuardrailEventHooks.post_call, None, {}, None, True), + (True, GuardrailEventHooks.pre_call, None, {}, None, False), + (False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {}, None, False), + (False, GuardrailEventHooks.pre_call, None, {"litellm_metadata": {"guardrails": ["pii-guard"]}}, None, False), + (False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {"litellm_metadata": {"guardrails": []}}, None, False), + (True, GuardrailEventHooks.pre_call, None, {"stream": True}, "streaming", False), + (True, GuardrailEventHooks.pre_call, None, {}, "streaming", True), + (True, GuardrailEventHooks.pre_call, None, {"stream": True}, "non_streaming", True), + (True, GuardrailEventHooks.pre_call, None, {}, "non_streaming", False), + ], + ids=[ + "guardrail-not-requested", + "guardrail-not-registered", + "default-on-post-call-guardrail", + "default-on-pre-call-guardrail", + "guardrail-on-the-route", + "guardrail-in-litellm-metadata", + "route-guardrail-beside-an-empty-client-list", + "streaming-guardrail-on-a-stream", + "streaming-guardrail-off-a-stream", + "non-streaming-guardrail-on-a-stream", + "non-streaming-guardrail-off-a-stream", + ], +) +async def test_pass_through_keeps_the_user_id_from_routes_a_guardrail_scans( + default_on: bool, + event_hook: GuardrailEventHooks, + guardrails_config: dict[str, object] | None, + client_fields: Mapping[str, object], + stream_scope: str | None, + forwarded: bool, +) -> None: + guardrail: Final = CustomGuardrail( + guardrail_name="pii-guard", default_on=default_on, event_hook=event_hook, stream_scope=stream_scope + ) + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}, **client_fields}) + body_without_client_fields: Final = {k: v for k, v in client_body.items() if k not in ("metadata", "litellm_metadata")} + + with patch.object(litellm, "callbacks", [guardrail]): + relayed: Final = await _relay_through_pass_through( + client_body, _ANTHROPIC_MESSAGES, guardrails_config=guardrails_config + ) + + assert relayed == ( + {**body_without_client_fields, "metadata": {"user_id": "user-1"}} if forwarded else body_without_client_fields + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +async def test_pass_through_warns_without_the_value_when_it_drops_a_refused_user_id( + caplog: pytest.LogCaptureFixture, +) -> None: + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "person@example.com"}}) + + with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): + await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None) + + assert "Not forwarding metadata.user_id" in caplog.text + assert "person@example.com" not in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +async def test_pass_through_drops_metadata_that_is_not_an_object() -> None: + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": "user-1"}) + + relayed: Final = await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None) + + assert relayed == _MESSAGES_BODY + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +async def test_pass_through_custom_body_replaces_the_client_user_id() -> None: + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}}) + + relayed: Final = await _relay_through_pass_through( + client_body, _ANTHROPIC_MESSAGES, guardrails_config=None, custom_body=dict(_MESSAGES_BODY) + ) + + assert relayed == _MESSAGES_BODY + + _PARTIAL_RELAY_WARNING_MARKER = "ended before upstream body was fully relayed" diff --git a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py index 5013ecd98a2..26ee9f04d01 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py @@ -996,3 +996,21 @@ async def test_guardrails_only_and_skip_guardrails_are_mutually_exclusive( guardrails_only=True, skip_guardrails=True, ) + + +@pytest.mark.parametrize( + "request_data, expected", + [ + ({"metadata": {"_guardrail_pipelines": ["p1"]}}, True), + ({"metadata": {"user_id": "u"}, "litellm_metadata": {"_guardrail_pipelines": ["p1"]}}, True), + ({"metadata": {"_guardrail_pipelines": 5}}, True), + ({"metadata": {"user_id": "u"}}, False), + ], + ids=["pipeline-in-metadata", "pipeline-in-litellm-metadata", "non-list-pipeline-value", "no-pipeline"], +) +def test_request_has_pre_call_guardrails_reads_pipelines_from_both_metadata_buckets( + proxy_logging, monkeypatch, request_data, expected +): + monkeypatch.setattr(litellm, "callbacks", []) + + assert proxy_logging.request_has_pre_call_guardrails(request_data) is expected