mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-01 07:03:02 +00:00
fix(proxy): forward a valid metadata.user_id to Anthropic Messages on pass-through routes (#49333)
* fix(proxy): forward the client's own metadata on Anthropic and custom pass-through routes * fix(proxy): forward only a valid metadata.user_id to Anthropic Messages on pass-through routes The whole-object forward sent keys Anthropic refuses and reached other providers, so a request that used to pass could start failing with a 400. Read the client's user_id after hooks, drop it unless it is a string with no email or phone number, and send only that key when the upstream URL path ends in /v1/messages. Share the validator with the translated route, whose email check now matches any embedded address as Anthropic's does. * fix(proxy): forward metadata.user_id only to Anthropic upstreams no guardrail scans Limit the pass-through user_id to Messages create on api.anthropic.com or the configured Anthropic base, mirror the checks Anthropic answers 400 for, and keep the field off any request a pre-call guardrail inspects. The translated route goes back to its own user_id filter. * fix(proxy): judge user_id forwarding on the whole request and cap it for configured Anthropic bases * refactor(proxy): pass the user_id check a plain mapping instead of an optional body * fix(proxy): drop an unprintable metadata.user_id and warn when a refused id is dropped Anthropic's whitespace is not Python's, so a control character before the at sign hid an email from the filter and turned a 200 into a 400. Fireworks and Moonshot also fail on newline ids. Requiring a printable id covers both, and a refused id now logs a warning that never carries the value. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
mika-lt
GitHub
parent
af070e7cd0
commit
e5aa3d32cd
9 changed files with 479 additions and 14 deletions
|
|
@ -634,6 +634,8 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60
|
|||
BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
|
||||
# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
|
||||
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024
|
||||
ANTHROPIC_MAX_USER_ID_LENGTH: Final = 512
|
||||
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH: Final = 255
|
||||
ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:"
|
||||
REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
|
||||
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ This file contains common utils for anthropic calls.
|
|||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Container, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -19,6 +20,8 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
|
||||
import litellm
|
||||
from litellm.constants import (
|
||||
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH,
|
||||
ANTHROPIC_MAX_USER_ID_LENGTH,
|
||||
DEFAULT_MODEL_CREATED_AT_TIME,
|
||||
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
|
||||
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
|
||||
|
|
@ -178,6 +181,40 @@ def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: boo
|
|||
return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools]
|
||||
|
||||
|
||||
def anthropic_pass_through_api_base() -> str:
|
||||
return os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
|
||||
|
||||
|
||||
def _same_host_and_port(url: httpx.URL, other: str) -> bool:
|
||||
try:
|
||||
other_url: Final = httpx.URL(other)
|
||||
except httpx.InvalidURL:
|
||||
return False
|
||||
return (url.host, url.port) == (other_url.host, other_url.port)
|
||||
|
||||
|
||||
def anthropic_upstream_user_id_limit(url: str) -> int | None:
|
||||
upstream: Final = httpx.URL(url)
|
||||
if not upstream.path.removesuffix("/").endswith("/v1/messages"):
|
||||
return None
|
||||
if upstream.host == "api.anthropic.com":
|
||||
return ANTHROPIC_MAX_USER_ID_LENGTH
|
||||
return (
|
||||
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH
|
||||
if _same_host_and_port(upstream, anthropic_pass_through_api_base())
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
def _has_email_shape(token: str) -> bool:
|
||||
at_sign: Final = token.find("@", 1)
|
||||
return at_sign != -1 and token.find(".", at_sign + 2, -1) != -1
|
||||
|
||||
|
||||
def anthropic_accepts_user_id(user_id: str, max_length: int) -> bool:
|
||||
return len(user_id) <= max_length and user_id.isprintable() and not any(map(_has_email_shape, user_id.split()))
|
||||
|
||||
|
||||
def is_anthropic_messages_url(url: str) -> bool:
|
||||
"""Check whether a URL addresses Anthropic's Messages API."""
|
||||
parsed_url: Final = urlparse(url)
|
||||
|
|
|
|||
|
|
@ -52,7 +52,11 @@ from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
|
|||
from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention
|
||||
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
|
||||
from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
AnthropicModelInfo,
|
||||
anthropic_pass_through_api_base,
|
||||
merge_anthropic_beta_headers,
|
||||
)
|
||||
from litellm.llms.azure.passthrough.transformation import (
|
||||
foreign_azure_deployment,
|
||||
is_azure_body_model_inference_endpoint,
|
||||
|
|
@ -1047,7 +1051,7 @@ async def anthropic_proxy_route(
|
|||
"""
|
||||
[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)
|
||||
"""
|
||||
base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
|
||||
base_target_url = anthropic_pass_through_api_base()
|
||||
encoded_endpoint = httpx.URL(endpoint).path
|
||||
|
||||
# Ensure endpoint starts with '/' for proper URL construction
|
||||
|
|
|
|||
|
|
@ -71,7 +71,11 @@ from litellm.litellm_core_utils.litellm_logging import get_masked_values
|
|||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.llms.anthropic.common_utils import is_anthropic_messages_url
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
anthropic_accepts_user_id,
|
||||
anthropic_upstream_user_id_limit,
|
||||
is_anthropic_messages_url,
|
||||
)
|
||||
from litellm.llms.base_llm.managed_resources.utils import (
|
||||
resolve_passthrough_managed_id_provider,
|
||||
)
|
||||
|
|
@ -503,7 +507,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
url: httpx.URL,
|
||||
headers: dict,
|
||||
requested_query_params: dict | None = None,
|
||||
_parsed_body: dict | None = None,
|
||||
_parsed_body: Mapping[str, object] | None = None,
|
||||
forward_multipart: bool = False,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
|
|
@ -717,8 +721,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
# unenforced. Set after the client merge so a request body cannot supply its own budget.
|
||||
#
|
||||
# Only for the built-in provider routes. `get_model_from_request` returns
|
||||
# None for a user-defined pass-through, deliberately: its body is forwarded
|
||||
# verbatim, so `model` there names an UPSTREAM model rather than a
|
||||
# None for a user-defined pass-through, deliberately: the client's `model`
|
||||
# goes upstream unchanged, so it names an UPSTREAM model rather than a
|
||||
# LiteLLM-managed one. Enforcement is therefore skipped on those routes, and
|
||||
# charging a counter anyway would track spend that nothing can refuse, and
|
||||
# would attribute it to a budget the operator scoped to a LiteLLM model that
|
||||
|
|
@ -1187,6 +1191,36 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
|
|||
return headers
|
||||
|
||||
|
||||
def _json_object(value: object) -> Mapping[str, object]:
|
||||
return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({}) # cast-ok: json
|
||||
|
||||
|
||||
def _anthropic_user_id_to_forward(
|
||||
body: Mapping[str, object],
|
||||
url: str,
|
||||
has_pre_call_guardrails: Callable[[Mapping[str, object]], bool],
|
||||
) -> str | None:
|
||||
user_id: Final = _json_object(body.get("metadata")).get("user_id")
|
||||
if not isinstance(user_id, str):
|
||||
return None
|
||||
max_length: Final = anthropic_upstream_user_id_limit(url)
|
||||
if max_length is None:
|
||||
return None
|
||||
if not anthropic_accepts_user_id(user_id, max_length):
|
||||
verbose_proxy_logger.warning("Not forwarding metadata.user_id because the upstream would refuse it")
|
||||
return None
|
||||
if has_pre_call_guardrails(body):
|
||||
verbose_proxy_logger.debug("Not forwarding metadata.user_id because a pre-call guardrail cannot scan it")
|
||||
return None
|
||||
return user_id
|
||||
|
||||
|
||||
def _with_anthropic_user_id(body: Mapping[str, object] | None, user_id: str | None) -> Mapping[str, object] | None:
|
||||
if body is None or user_id is None:
|
||||
return body
|
||||
return {**body, "metadata": {"user_id": user_id}}
|
||||
|
||||
|
||||
def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None
|
||||
|
||||
|
|
@ -1423,6 +1457,10 @@ async def pass_through_request(
|
|||
request_method=getattr(request, "method", None),
|
||||
cost_per_request=cost_per_request,
|
||||
)
|
||||
hooked_body: Final = _json_object(_parsed_body)
|
||||
client_user_id: Final = _anthropic_user_id_to_forward(
|
||||
hooked_body, str(url), proxy_logging_obj.request_has_pre_call_guardrails
|
||||
)
|
||||
kwargs = HttpPassThroughEndpointHelpers.init_kwargs_for_pass_through_endpoint( # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
_parsed_body=_parsed_body,
|
||||
|
|
@ -1590,11 +1628,13 @@ async def pass_through_request(
|
|||
else:
|
||||
logging_url = str(url) + "?" + requested_query_params_str
|
||||
|
||||
outgoing_body: Final = cast("Mapping[str, object] | None", _parsed_body) # cast-ok: json
|
||||
upstream_body: Final = _with_anthropic_user_id(outgoing_body, client_user_id)
|
||||
logging_obj.pre_call(
|
||||
input=[{"role": "user", "content": safe_dumps(_parsed_body)}],
|
||||
api_key="",
|
||||
additional_args={
|
||||
"complete_input_dict": _parsed_body,
|
||||
"complete_input_dict": upstream_body,
|
||||
"api_base": str(logging_url),
|
||||
"headers": redact_credential_headers(upstream_headers),
|
||||
},
|
||||
|
|
@ -1634,7 +1674,7 @@ async def pass_through_request(
|
|||
url,
|
||||
params=requested_query_params,
|
||||
headers=upstream_headers,
|
||||
json=_parsed_body,
|
||||
json=upstream_body,
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -1733,7 +1773,7 @@ async def pass_through_request(
|
|||
url=url,
|
||||
headers=upstream_headers,
|
||||
requested_query_params=requested_query_params,
|
||||
_parsed_body=_parsed_body,
|
||||
_parsed_body=upstream_body,
|
||||
forward_multipart=is_multipart,
|
||||
)
|
||||
verbose_proxy_logger.debug("response.headers= %s", response.headers)
|
||||
|
|
|
|||
|
|
@ -3031,14 +3031,21 @@ class ProxyLogging:
|
|||
hook to consult, but it judges the payload the same way, so a proxy configured only with
|
||||
one of those still has something to say about every record.
|
||||
"""
|
||||
if request_metadata.get("_guardrail_pipelines"):
|
||||
return self.request_has_pre_call_guardrails({"metadata": dict(request_metadata)}, event_type)
|
||||
|
||||
def request_has_pre_call_guardrails(
|
||||
self,
|
||||
request_data: Mapping[str, object],
|
||||
event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call,
|
||||
) -> bool:
|
||||
if _policy_state_metadata(request_data).get("_guardrail_pipelines"):
|
||||
return True
|
||||
caps: Final = ProxyLogging.callback_capabilities()
|
||||
if caps.has_content_enforcer:
|
||||
return True
|
||||
probe: Final = {"metadata": dict(request_metadata)}
|
||||
return any(
|
||||
isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type)
|
||||
isinstance(callback, CustomGuardrail)
|
||||
and callback.should_run_guardrail(data=request_data, event_type=event_type)
|
||||
for callback in caps.resolved_callbacks
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -317,7 +317,6 @@ def test_credentialless_proxy_forwards_the_caller_anthropic_credential_and_never
|
|||
|
||||
|
||||
def test_anthropic_sdk_metadata_user_id_reaches_upstream(api_key_proxy: Gateway, wire: Wire) -> None:
|
||||
pytest.skip("BUG: /anthropic/v1/messages drops the native metadata.user_id field before forwarding upstream")
|
||||
with api_key_proxy.scenario() as scenario:
|
||||
key: Final = scenario.key()
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
|
|
|||
|
|
@ -56,6 +56,98 @@ def test_is_anthropic_messages_url(url: str, expected: bool) -> None:
|
|||
assert is_anthropic_messages_url(url) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured_base", "url", "expected"),
|
||||
[
|
||||
pytest.param(None, "https://api.anthropic.com/v1/messages", 512, id="default-upstream"),
|
||||
pytest.param(None, "https://api.anthropic.com:443/v1/messages/", 512, id="default-upstream-explicit-port"),
|
||||
pytest.param("http://relay.test:8080", "https://api.anthropic.com/v1/messages", 512, id="direct-target"),
|
||||
pytest.param("https://api.anthropic.com", "https://api.anthropic.com/v1/messages", 512, id="configured-default"),
|
||||
pytest.param("http://relay.test:8080", "http://relay.test:8080/v1/messages", 255, id="configured-upstream"),
|
||||
pytest.param("http://relay.test:8080/a", "http://relay.test:8080/a/v1/messages", 255, id="configured-prefix"),
|
||||
pytest.param("https://Relay.Test", "https://relay.test/v1/messages", 255, id="configured-upper-case-host"),
|
||||
pytest.param("https://relay.test:443", "https://relay.test/v1/messages", 255, id="configured-https-port"),
|
||||
pytest.param("http://relay.test:80/a", "http://relay.test/a/v1/messages", 255, id="configured-http-port"),
|
||||
pytest.param("http://relay.test:8080", "http://relay.test:8081/v1/messages", None, id="other-port"),
|
||||
pytest.param("http://relay.test:notaport", "http://relay.test/v1/messages", None, id="invalid-configured-base"),
|
||||
pytest.param(None, "https://other-vendor.test/v1/messages", None, id="other-vendor"),
|
||||
pytest.param(None, "https://api.anthropic.com/v1/messages/count_tokens", None, id="count-tokens"),
|
||||
],
|
||||
)
|
||||
def test_anthropic_upstream_user_id_limit(
|
||||
monkeypatch: pytest.MonkeyPatch, configured_base: str | None, url: str, expected: int | None
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.common_utils import anthropic_upstream_user_id_limit
|
||||
|
||||
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
|
||||
if configured_base is not None:
|
||||
monkeypatch.setenv("ANTHROPIC_API_BASE", configured_base)
|
||||
|
||||
assert anthropic_upstream_user_id_limit(url) == expected
|
||||
|
||||
|
||||
def test_anthropic_pass_through_api_base_prefers_the_api_base_variable(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.llms.anthropic.common_utils import anthropic_pass_through_api_base
|
||||
|
||||
monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://fallback.test")
|
||||
monkeypatch.setenv("ANTHROPIC_API_BASE", "http://preferred.test")
|
||||
assert anthropic_pass_through_api_base() == "http://preferred.test"
|
||||
|
||||
monkeypatch.delenv("ANTHROPIC_API_BASE")
|
||||
assert anthropic_pass_through_api_base() == "http://fallback.test"
|
||||
|
||||
|
||||
# Each row was sent to https://api.anthropic.com/v1/messages on 2026-10-30, False rows answered 400.
|
||||
@pytest.mark.parametrize(
|
||||
("user_id", "expected"),
|
||||
[
|
||||
pytest.param("user-1", True, id="opaque"),
|
||||
pytest.param("", True, id="empty"),
|
||||
pytest.param('{"device_id":"abc","session_id":"def"}', True, id="claude-code-shape"),
|
||||
pytest.param("1234567890", True, id="digits"),
|
||||
pytest.param("+14155550100", True, id="phone-shape"),
|
||||
pytest.param("a@b", True, id="no-dot"),
|
||||
pytest.param("@b.co", True, id="nothing-before-at-sign"),
|
||||
pytest.param("a@.co", True, id="nothing-between-at-sign-and-dot"),
|
||||
pytest.param("a@b.", True, id="nothing-after-dot"),
|
||||
pytest.param("a @ b.co", True, id="spaces-around-at-sign"),
|
||||
pytest.param("a\u00a0@b.co", False, id="no-break-space-before-at-sign"),
|
||||
pytest.param("a@b\n.co", False, id="newline-before-dot"),
|
||||
pytest.param("line one\nline two", False, id="newline"),
|
||||
pytest.param("a\tb", False, id="tab"),
|
||||
pytest.param("a\x1f@b.co", False, id="unit-separator-before-at-sign"),
|
||||
pytest.param("a@\x1cb.co", False, id="file-separator-after-at-sign"),
|
||||
pytest.param("a\u200bb", False, id="zero-width-space"),
|
||||
pytest.param("a@b.co", False, id="email"),
|
||||
pytest.param("user a@b.co here", False, id="email-inside-text"),
|
||||
pytest.param('{"device_id":"a@b.co"}', False, id="email-inside-json"),
|
||||
pytest.param("a@@example.com", False, id="doubled-at-sign"),
|
||||
pytest.param("@@example.com", False, id="leading-doubled-at-sign"),
|
||||
pytest.param("a@b@c.co", False, id="two-at-signs"),
|
||||
pytest.param("a@b..co", False, id="doubled-dot"),
|
||||
pytest.param("a@.b.co", False, id="dot-after-at-sign"),
|
||||
pytest.param("a@b.c d", False, id="email-before-space"),
|
||||
],
|
||||
)
|
||||
def test_anthropic_accepts_user_id(user_id: str, expected: bool) -> None:
|
||||
from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH
|
||||
from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id
|
||||
|
||||
assert anthropic_accepts_user_id(user_id, ANTHROPIC_MAX_USER_ID_LENGTH) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("character", ["a", "\u00e9", "\U0001f600"], ids=["ascii", "two-bytes", "four-bytes"])
|
||||
def test_anthropic_accepts_user_id_counts_characters_up_to_the_limit(character: str) -> None:
|
||||
from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH
|
||||
from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id
|
||||
|
||||
assert anthropic_accepts_user_id(character * ANTHROPIC_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH) is True
|
||||
assert (
|
||||
anthropic_accepts_user_id(character * (ANTHROPIC_MAX_USER_ID_LENGTH + 1), ANTHROPIC_MAX_USER_ID_LENGTH) is False
|
||||
)
|
||||
|
||||
|
||||
def test_anthropic_model_not_found_maps_to_not_found_but_authentication_error_does_not() -> None:
|
||||
with respx.mock(assert_all_called=True) as router:
|
||||
router.post("https://api.anthropic.com/v1/messages").mock(
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
|
||||
from litellm.constants import ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH, DEFAULT_REQUEST_TIMEOUT_SECONDS
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
|
|
@ -6621,6 +6621,272 @@ async def test_pass_through_body_stays_byte_identical_outside_the_opt_in(
|
|||
assert "x-litellm-call-id" in response.headers
|
||||
|
||||
|
||||
def _echo_upstream_body(upstream_request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200, content=upstream_request.content, headers={"content-type": "application/json"}, request=upstream_request
|
||||
)
|
||||
|
||||
|
||||
async def _relayed_body(response: Response) -> bytes:
|
||||
if isinstance(response, StreamingResponse):
|
||||
return b"".join([chunk async for chunk in response.body_iterator])
|
||||
return bytes(response.body)
|
||||
|
||||
|
||||
async def _relay_through_pass_through(
|
||||
client_body: Mapping[str, object],
|
||||
target: str,
|
||||
guardrails_config: dict[str, object] | None,
|
||||
custom_body: dict[str, object] | None = None,
|
||||
) -> Mapping[str, object]:
|
||||
async def receive() -> Mapping[str, object]:
|
||||
return {"type": "http.request", "body": json.dumps(dict(client_body)).encode()}
|
||||
|
||||
request: Final = Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": "/my-claude-route",
|
||||
"query_string": b"",
|
||||
"headers": [(b"content-type", b"application/json")],
|
||||
},
|
||||
receive,
|
||||
)
|
||||
fake_client, cleanup = _inject_fake_passthrough_client(httpx.MockTransport(_echo_upstream_body), timeout=None)
|
||||
try:
|
||||
with ExitStack() as stack:
|
||||
_enter_relay_logging_mocks(stack, {})
|
||||
stack.enter_context(
|
||||
patch("litellm.proxy.proxy_server.proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache()))
|
||||
)
|
||||
response: Final = await pass_through_request(
|
||||
request=request,
|
||||
target=target,
|
||||
custom_headers={},
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"),
|
||||
custom_body=custom_body,
|
||||
guardrails_config=guardrails_config,
|
||||
)
|
||||
relayed: Final = await _relayed_body(response)
|
||||
finally:
|
||||
cleanup()
|
||||
await fake_client.aclose()
|
||||
|
||||
assert response.status_code == 200
|
||||
return MappingProxyType(json.loads(relayed))
|
||||
|
||||
|
||||
_MESSAGES_BODY: Final = MappingProxyType(
|
||||
{"model": "claude-sonnet-4-5", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]}
|
||||
)
|
||||
_ANTHROPIC_MESSAGES: Final = "https://api.anthropic.com/v1/messages"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_configured_anthropic_base(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"])
|
||||
@pytest.mark.parametrize(
|
||||
"client_metadata, target, forwarded_metadata",
|
||||
[
|
||||
({"user_id": "user-1"}, _ANTHROPIC_MESSAGES, {"user_id": "user-1"}),
|
||||
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/", {"user_id": "user-1"}),
|
||||
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}?beta=true", {"user_id": "user-1"}),
|
||||
(
|
||||
{"user_id": "user-1", "tags": ["a"], "trace_id": "t", "generation_name": "g", "foo": "bar"},
|
||||
_ANTHROPIC_MESSAGES,
|
||||
{"user_id": "user-1"},
|
||||
),
|
||||
({"user_id": "1234567890"}, _ANTHROPIC_MESSAGES, {"user_id": "1234567890"}),
|
||||
({"tags": ["a"], "foo": "bar"}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": "a@example.com"}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": "x" * (ANTHROPIC_MAX_USER_ID_LENGTH + 1)}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": 42}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": "line one\nline two"}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": "a\x1f@example.com"}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/count_tokens", None),
|
||||
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/batches", None),
|
||||
({"user_id": "user-1"}, "https://api.anthropic.com/v1/models", None),
|
||||
({"user_id": "user-1"}, "https://other-vendor.test/anthropic/v1/messages", None),
|
||||
({"user_id": "user-1"}, "https://other-vendor.test/v1/responses", None),
|
||||
({"user_id": "user-1"}, "https://other-vendor.test/v1/chat/completions", None),
|
||||
],
|
||||
ids=[
|
||||
"messages",
|
||||
"messages-trailing-slash",
|
||||
"messages-with-query",
|
||||
"user-id-beside-litellm-and-unknown-keys",
|
||||
"digits-user-id",
|
||||
"no-user-id",
|
||||
"email-user-id",
|
||||
"over-long-user-id",
|
||||
"integer-user-id",
|
||||
"newline-user-id",
|
||||
"control-character-before-at-sign",
|
||||
"count-tokens",
|
||||
"batches",
|
||||
"other-anthropic-path",
|
||||
"other-vendor-messages",
|
||||
"other-vendor-responses",
|
||||
"other-vendor-chat-completions",
|
||||
],
|
||||
)
|
||||
async def test_pass_through_forwards_only_an_acceptable_user_id_to_anthropic_messages(
|
||||
stream: bool, client_metadata: object, target: str, forwarded_metadata: Mapping[str, object] | None
|
||||
) -> None:
|
||||
client_body: Final = MappingProxyType(
|
||||
{**_MESSAGES_BODY, "metadata": client_metadata, **({"stream": True} if stream else {})}
|
||||
)
|
||||
body_without_metadata: Final = {k: v for k, v in client_body.items() if k != "metadata"}
|
||||
|
||||
relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None)
|
||||
|
||||
assert relayed == (
|
||||
body_without_metadata if forwarded_metadata is None else {**body_without_metadata, "metadata": forwarded_metadata}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
@pytest.mark.parametrize("base_env", ["ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"])
|
||||
@pytest.mark.parametrize(
|
||||
"target, forwarded",
|
||||
[
|
||||
("http://gateway.test:8080/anthropic/v1/messages", True),
|
||||
("http://gateway.test:9090/v1/messages", False),
|
||||
("http://other-gateway.test:8080/v1/messages", False),
|
||||
],
|
||||
ids=["configured-upstream", "other-port", "other-host"],
|
||||
)
|
||||
async def test_pass_through_treats_the_configured_anthropic_base_as_anthropic(
|
||||
monkeypatch: pytest.MonkeyPatch, base_env: str, target: str, forwarded: bool
|
||||
) -> None:
|
||||
monkeypatch.setenv(base_env, "http://gateway.test:8080/anthropic")
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}})
|
||||
|
||||
relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None)
|
||||
|
||||
assert relayed == (client_body if forwarded else _MESSAGES_BODY)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
@pytest.mark.parametrize(
|
||||
"user_id_length, target, forwarded",
|
||||
[
|
||||
(ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, "http://gateway.test/v1/messages", True),
|
||||
(ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH + 1, "http://gateway.test/v1/messages", False),
|
||||
(ANTHROPIC_MAX_USER_ID_LENGTH, _ANTHROPIC_MESSAGES, True),
|
||||
],
|
||||
ids=["at-the-configured-base-limit", "over-the-configured-base-limit", "direct-anthropic-keeps-its-limit"],
|
||||
)
|
||||
async def test_pass_through_caps_the_user_id_for_a_configured_anthropic_compatible_base(
|
||||
monkeypatch: pytest.MonkeyPatch, user_id_length: int, target: str, forwarded: bool
|
||||
) -> None:
|
||||
monkeypatch.setenv("ANTHROPIC_API_BASE", "http://gateway.test")
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "u" * user_id_length}})
|
||||
|
||||
relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None)
|
||||
|
||||
assert relayed == (client_body if forwarded else _MESSAGES_BODY)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
@pytest.mark.parametrize(
|
||||
"default_on, event_hook, guardrails_config, client_fields, stream_scope, forwarded",
|
||||
[
|
||||
(False, GuardrailEventHooks.pre_call, None, {}, None, True),
|
||||
(False, GuardrailEventHooks.pre_call, {"unregistered-guard": {}}, {}, None, True),
|
||||
(True, GuardrailEventHooks.post_call, None, {}, None, True),
|
||||
(True, GuardrailEventHooks.pre_call, None, {}, None, False),
|
||||
(False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {}, None, False),
|
||||
(False, GuardrailEventHooks.pre_call, None, {"litellm_metadata": {"guardrails": ["pii-guard"]}}, None, False),
|
||||
(False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {"litellm_metadata": {"guardrails": []}}, None, False),
|
||||
(True, GuardrailEventHooks.pre_call, None, {"stream": True}, "streaming", False),
|
||||
(True, GuardrailEventHooks.pre_call, None, {}, "streaming", True),
|
||||
(True, GuardrailEventHooks.pre_call, None, {"stream": True}, "non_streaming", True),
|
||||
(True, GuardrailEventHooks.pre_call, None, {}, "non_streaming", False),
|
||||
],
|
||||
ids=[
|
||||
"guardrail-not-requested",
|
||||
"guardrail-not-registered",
|
||||
"default-on-post-call-guardrail",
|
||||
"default-on-pre-call-guardrail",
|
||||
"guardrail-on-the-route",
|
||||
"guardrail-in-litellm-metadata",
|
||||
"route-guardrail-beside-an-empty-client-list",
|
||||
"streaming-guardrail-on-a-stream",
|
||||
"streaming-guardrail-off-a-stream",
|
||||
"non-streaming-guardrail-on-a-stream",
|
||||
"non-streaming-guardrail-off-a-stream",
|
||||
],
|
||||
)
|
||||
async def test_pass_through_keeps_the_user_id_from_routes_a_guardrail_scans(
|
||||
default_on: bool,
|
||||
event_hook: GuardrailEventHooks,
|
||||
guardrails_config: dict[str, object] | None,
|
||||
client_fields: Mapping[str, object],
|
||||
stream_scope: str | None,
|
||||
forwarded: bool,
|
||||
) -> None:
|
||||
guardrail: Final = CustomGuardrail(
|
||||
guardrail_name="pii-guard", default_on=default_on, event_hook=event_hook, stream_scope=stream_scope
|
||||
)
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}, **client_fields})
|
||||
body_without_client_fields: Final = {k: v for k, v in client_body.items() if k not in ("metadata", "litellm_metadata")}
|
||||
|
||||
with patch.object(litellm, "callbacks", [guardrail]):
|
||||
relayed: Final = await _relay_through_pass_through(
|
||||
client_body, _ANTHROPIC_MESSAGES, guardrails_config=guardrails_config
|
||||
)
|
||||
|
||||
assert relayed == (
|
||||
{**body_without_client_fields, "metadata": {"user_id": "user-1"}} if forwarded else body_without_client_fields
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
async def test_pass_through_warns_without_the_value_when_it_drops_a_refused_user_id(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "person@example.com"}})
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name):
|
||||
await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None)
|
||||
|
||||
assert "Not forwarding metadata.user_id" in caplog.text
|
||||
assert "person@example.com" not in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
async def test_pass_through_drops_metadata_that_is_not_an_object() -> None:
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": "user-1"})
|
||||
|
||||
relayed: Final = await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None)
|
||||
|
||||
assert relayed == _MESSAGES_BODY
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
async def test_pass_through_custom_body_replaces_the_client_user_id() -> None:
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}})
|
||||
|
||||
relayed: Final = await _relay_through_pass_through(
|
||||
client_body, _ANTHROPIC_MESSAGES, guardrails_config=None, custom_body=dict(_MESSAGES_BODY)
|
||||
)
|
||||
|
||||
assert relayed == _MESSAGES_BODY
|
||||
|
||||
|
||||
_PARTIAL_RELAY_WARNING_MARKER = "ended before upstream body was fully relayed"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -996,3 +996,21 @@ async def test_guardrails_only_and_skip_guardrails_are_mutually_exclusive(
|
|||
guardrails_only=True,
|
||||
skip_guardrails=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_data, expected",
|
||||
[
|
||||
({"metadata": {"_guardrail_pipelines": ["p1"]}}, True),
|
||||
({"metadata": {"user_id": "u"}, "litellm_metadata": {"_guardrail_pipelines": ["p1"]}}, True),
|
||||
({"metadata": {"_guardrail_pipelines": 5}}, True),
|
||||
({"metadata": {"user_id": "u"}}, False),
|
||||
],
|
||||
ids=["pipeline-in-metadata", "pipeline-in-litellm-metadata", "non-list-pipeline-value", "no-pipeline"],
|
||||
)
|
||||
def test_request_has_pre_call_guardrails_reads_pipelines_from_both_metadata_buckets(
|
||||
proxy_logging, monkeypatch, request_data, expected
|
||||
):
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
|
||||
assert proxy_logging.request_has_pre_call_guardrails(request_data) is expected
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue