fix(proxy): keep key guardrails, metadata type errors and failure-row client details on Responses-style bodies

This commit is contained in:
mateo-berri 2026-10-29 19:24:35 -07:00
parent acd64a7d36
commit c3789474dc
7 changed files with 160 additions and 23 deletions

View file

@ -9,6 +9,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args
import httpx
from pydantic import TypeAdapter
from litellm._internal_context import with_service_target
from litellm._logging import verbose_logger
@ -218,6 +219,15 @@ def _request_is_streaming(data: object, event_type: GuardrailEventHooks | None =
)
_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object])
def _guardrails_listed_in(metadata: object) -> list[str | dict[str, DynamicGuardrailParams]]:
if not isinstance(metadata, dict):
return []
return metadata.get("guardrails") or []
class CustomGuardrail(CustomLogger):
# If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
use_native_during_call_hook: ClassVar[bool] = False
@ -791,28 +801,22 @@ class CustomGuardrail(CustomLogger):
return True
raise
def get_guardrail_from_metadata(self, data: dict) -> list[str] | list[dict[str, DynamicGuardrailParams]]:
def get_guardrail_from_metadata(self, data: dict) -> list[str | dict[str, DynamicGuardrailParams]]:
"""
Returns the guardrail(s) to be run from the metadata or root
Returns the guardrail(s) to be run from the root, else from both metadata fields together
"""
if "guardrails" in data:
return data["guardrails"]
# Check both metadata locations. For regular endpoints move_guardrails_to_metadata
# writes to "metadata"; for thread/assistant endpoints it writes to
# "litellm_metadata". We check the one that actually contains the "guardrails"
# key so that a non-empty litellm_metadata without guardrails does not shadow
# the merged list stored in metadata (which would cause team guardrails to be
# silently skipped while default_on=True policy guardrails still fire).
for meta_key in ("metadata", "litellm_metadata"):
meta = data.get(meta_key) or {}
if isinstance(meta, dict) and "guardrails" in meta:
return meta.get("guardrails") or []
return []
request_fields: Final = _REQUEST_FIELDS.validate_python(data)
return [
*_guardrails_listed_in(request_fields.get("metadata")),
*_guardrails_listed_in(request_fields.get("litellm_metadata")),
]
def _guardrail_is_in_requested_guardrails(
self,
requested_guardrails: list[str] | list[dict[str, DynamicGuardrailParams]],
requested_guardrails: Sequence[str | dict[str, DynamicGuardrailParams]],
) -> bool:
for _guardrail in requested_guardrails:
if isinstance(_guardrail, dict):

View file

@ -1367,12 +1367,15 @@ async def common_checks(
if not skip_all_budget_checks:
# Key metadata.tags are injected into request_body here so the tag budget
# check can read them; this mutation must run before the gathered checks.
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_request
if valid_token is not None:
for tag_target in LiteLLMProxyRequestSetup.pre_auth_tag_merge_targets(request, request_body):
LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route(request_data=tag_target, route=route)
LiteLLMProxyRequestSetup.merge_caller_body_tags(request_data=tag_target)
LiteLLMProxyRequestSetup.merge_caller_body_tags(
request_data=tag_target,
metadata_variable_name=metadata_variable_name_for_request(route, tag_target),
)
LiteLLMProxyRequestSetup.apply_key_tags_pre_auth(request_data=tag_target, user_api_key_dict=valid_token)
LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, request_body)

View file

@ -137,19 +137,34 @@ _CAPTURED_IDENTITY_CALL_TYPES: Final[frozenset[str]] = frozenset(
_OBJECT_MAPPING: Final = TypeAdapter(ObjectMapping)
def _proxy_stamped_used_client_oauth_token(
request_data: Mapping[str, object], request_route: str | None
) -> bool | None:
def _proxy_stamped_metadata(request_data: Mapping[str, object], request_route: str | None) -> Mapping[str, object]:
proxy_bucket: Final = (
get_metadata_variable_name_from_kwargs(request_data)
if request_route is None
else metadata_variable_name_for_request(request_route, request_data)
)
proxy_metadata: Final = request_data.get(proxy_bucket)
stamped: Final = proxy_metadata.get("used_client_oauth_token") if isinstance(proxy_metadata, dict) else None
return _str_keyed_fields(request_data.get(proxy_bucket))
def _proxy_stamped_used_client_oauth_token(
request_data: Mapping[str, object], request_route: str | None
) -> bool | None:
stamped: Final = _proxy_stamped_metadata(request_data, request_route).get("used_client_oauth_token")
return stamped if isinstance(stamped, bool) else None
_PROXY_STAMPED_REQUEST_FIELDS: Final = ("requester_ip_address", "user_agent")
def _proxy_stamped_request_fields(
request_data: Mapping[str, object], request_route: str | None
) -> Mapping[str, object]:
if request_route is None:
return {}
proxy_metadata: Final = _proxy_stamped_metadata(request_data, request_route)
return {field: proxy_metadata[field] for field in _PROXY_STAMPED_REQUEST_FIELDS if field in proxy_metadata}
def _str_keyed_fields(value: object) -> Mapping[str, object]:
return value if is_str_keyed_mapping(value) else {}
@ -455,7 +470,13 @@ class _ProxyDBLogger(CustomLogger):
_metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_fields, request_route)
existing_metadata: Final[dict] = request_data.get("metadata", None) or {}
existing_metadata.update({**_routed_metadata_fallback(request_fields), **_metadata})
existing_metadata.update(
{
**_routed_metadata_fallback(request_fields),
**_proxy_stamped_request_fields(request_fields, request_route),
**_metadata,
}
)
litellm_metadata_bucket: Final = request_data.get("litellm_metadata")
litellm_metadata: Final[ObjectMapping] = (

View file

@ -655,6 +655,12 @@ def normalized_metadata_slot(
return normalized
def _holds_metadata_object(value: object) -> bool:
if value is None or isinstance(value, dict):
return True
return isinstance(value, str) and isinstance(_DECODED_JSON.validate_python(safe_json_loads(value)), dict)
def _metadata_object_or_error(field: str, value: object) -> Mapping[str, object] | ProxyException:
if value is None:
return {}
@ -2429,7 +2435,7 @@ class LiteLLMProxyRequestSetup:
if metadata_variable_name is not None
else get_metadata_variable_name_from_kwargs(request_data)
)
if slot == "metadata":
if slot == "metadata" or not _holds_metadata_object(request_data.get(slot)):
return
raw_caller_metadata: Final = request_data.get("metadata")
caller_metadata: Final = (

View file

@ -3699,6 +3699,20 @@ def test_get_guardrail_from_metadata() -> None:
) == [{"test-guardrail": {"extra_body": {"key": "value"}}}]
@pytest.mark.parametrize(
"caller_metadata",
[{"guardrails": []}, {"guardrails": ["not-a-guardrail"]}, {"guardrails": None}],
)
def test_caller_metadata_guardrails_never_hide_the_proxy_guardrails_in_litellm_metadata(caller_metadata: dict) -> None:
guardrail: Final = CustomGuardrail(
guardrail_name="key-guardrail", event_hook=GuardrailEventHooks.pre_call, default_on=False
)
request_data: Final = {"metadata": caller_metadata, "litellm_metadata": {"guardrails": ["key-guardrail"]}}
assert guardrail.should_run_guardrail(request_data, GuardrailEventHooks.pre_call) is True
def test_guardrail_is_in_requested_guardrails() -> None:
guardrail: Final = CustomGuardrail(guardrail_name="test-guardrail")

View file

@ -623,6 +623,56 @@ async def test_async_post_call_failure_hook_reads_used_client_oauth_token_from_t
assert json.loads(payload["metadata"])["used_client_oauth_token"] is expected
@pytest.mark.asyncio
@pytest.mark.parametrize(
"request_route, body",
[
(
"/cursor/chat/completions",
{
"input": "Hello",
"metadata": {"requester_ip_address": "203.0.113.9", "user_agent": "spoofed"},
"litellm_metadata": {"requester_ip_address": "127.0.0.1", "user_agent": "Cursor/1.0"},
},
),
(
"/v1/chat/completions",
{
"messages": [{"role": "user", "content": "Hello"}],
"metadata": {"requester_ip_address": "127.0.0.1", "user_agent": "Cursor/1.0"},
"litellm_metadata": {"requester_ip_address": "203.0.113.9", "user_agent": "spoofed"},
},
),
],
)
async def test_async_post_call_failure_hook_logs_the_proxy_stamped_ip_and_user_agent(request_route: str, body: dict):
request_data = {
"model": "gpt-4o",
"proxy_server_request": {"request_id": "test_request_id"},
**body,
}
with patch(
"litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database",
new_callable=AsyncMock,
) as mock_update_database:
await ProxyDBLogger().async_post_call_failure_hook(
request_data=request_data,
original_exception=Exception("invalid api key"),
user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key", request_route=request_route),
)
payload = get_logging_payload(
kwargs=mock_update_database.call_args[1]["kwargs"],
response_obj={},
start_time=datetime.now(),
end_time=datetime.now(),
)
assert payload["requester_ip_address"] == "127.0.0.1"
assert json.loads(payload["metadata"])["user_agent"] == "Cursor/1.0"
@pytest.mark.asyncio
async def test_async_post_call_failure_hook_bills_guardrail_cost_on_blocked_request():
"""LIT-5651: a request blocked by a guardrail never reaches the LLM, but the

View file

@ -698,6 +698,45 @@ async def test_add_litellm_data_to_request_rejects_non_object_metadata(field, va
assert field not in data
@pytest.mark.asyncio
@pytest.mark.parametrize("route", ["/v1/chat/completions", "/v1/responses", "/cursor/chat/completions"])
@pytest.mark.parametrize("value, received_type", [("junk", "a string"), (5, "an integer"), ([1], "an array")])
async def test_invalid_litellm_metadata_answers_400_when_metadata_carries_tags(route, value, received_type):
from litellm.proxy.auth.auth_checks import common_checks
from litellm.proxy.utils import ProxyLogging
data = {"model": "gpt-4o", "input": "hi", "metadata": {"tags": ["client-tag"]}, "litellm_metadata": value}
request = _batches_request_mock()
await common_checks(
request_body=data,
team_object=None,
user_object=None,
end_user_object=None,
global_proxy_spend=None,
general_settings={},
route=route,
llm_router=None,
proxy_logging_obj=ProxyLogging(user_api_key_cache=None),
valid_token=UserAPIKeyAuth(api_key="hashed-key"),
request=request,
)
with pytest.raises(ProxyException) as exc_info:
await add_litellm_data_to_request(
data=data,
request=request,
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
proxy_config=MagicMock(),
general_settings={},
version="test-version",
)
assert exc_info.value.code == "400"
assert exc_info.value.message == (
f"Invalid type for 'litellm_metadata': expected an object, but got {received_type} instead."
)
@pytest.mark.asyncio
async def test_add_litellm_data_to_request_removes_every_invalid_metadata_field_before_raising():
"""When both fields are invalid, the raise for the first must not leave the