mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): keep key guardrails, metadata type errors and failure-row client details on Responses-style bodies
This commit is contained in:
parent
acd64a7d36
commit
c3789474dc
7 changed files with 160 additions and 23 deletions
|
|
@ -9,6 +9,7 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -218,6 +219,15 @@ def _request_is_streaming(data: object, event_type: GuardrailEventHooks | None =
|
|||
)
|
||||
|
||||
|
||||
_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _guardrails_listed_in(metadata: object) -> list[str | dict[str, DynamicGuardrailParams]]:
|
||||
if not isinstance(metadata, dict):
|
||||
return []
|
||||
return metadata.get("guardrails") or []
|
||||
|
||||
|
||||
class CustomGuardrail(CustomLogger):
|
||||
# If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
|
||||
use_native_during_call_hook: ClassVar[bool] = False
|
||||
|
|
@ -791,28 +801,22 @@ class CustomGuardrail(CustomLogger):
|
|||
return True
|
||||
raise
|
||||
|
||||
def get_guardrail_from_metadata(self, data: dict) -> list[str] | list[dict[str, DynamicGuardrailParams]]:
|
||||
def get_guardrail_from_metadata(self, data: dict) -> list[str | dict[str, DynamicGuardrailParams]]:
|
||||
"""
|
||||
Returns the guardrail(s) to be run from the metadata or root
|
||||
Returns the guardrail(s) to be run from the root, else from both metadata fields together
|
||||
"""
|
||||
|
||||
if "guardrails" in data:
|
||||
return data["guardrails"]
|
||||
# Check both metadata locations. For regular endpoints move_guardrails_to_metadata
|
||||
# writes to "metadata"; for thread/assistant endpoints it writes to
|
||||
# "litellm_metadata". We check the one that actually contains the "guardrails"
|
||||
# key so that a non-empty litellm_metadata without guardrails does not shadow
|
||||
# the merged list stored in metadata (which would cause team guardrails to be
|
||||
# silently skipped while default_on=True policy guardrails still fire).
|
||||
for meta_key in ("metadata", "litellm_metadata"):
|
||||
meta = data.get(meta_key) or {}
|
||||
if isinstance(meta, dict) and "guardrails" in meta:
|
||||
return meta.get("guardrails") or []
|
||||
return []
|
||||
request_fields: Final = _REQUEST_FIELDS.validate_python(data)
|
||||
return [
|
||||
*_guardrails_listed_in(request_fields.get("metadata")),
|
||||
*_guardrails_listed_in(request_fields.get("litellm_metadata")),
|
||||
]
|
||||
|
||||
def _guardrail_is_in_requested_guardrails(
|
||||
self,
|
||||
requested_guardrails: list[str] | list[dict[str, DynamicGuardrailParams]],
|
||||
requested_guardrails: Sequence[str | dict[str, DynamicGuardrailParams]],
|
||||
) -> bool:
|
||||
for _guardrail in requested_guardrails:
|
||||
if isinstance(_guardrail, dict):
|
||||
|
|
|
|||
|
|
@ -1367,12 +1367,15 @@ async def common_checks(
|
|||
if not skip_all_budget_checks:
|
||||
# Key metadata.tags are injected into request_body here so the tag budget
|
||||
# check can read them; this mutation must run before the gathered checks.
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_request
|
||||
|
||||
if valid_token is not None:
|
||||
for tag_target in LiteLLMProxyRequestSetup.pre_auth_tag_merge_targets(request, request_body):
|
||||
LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route(request_data=tag_target, route=route)
|
||||
LiteLLMProxyRequestSetup.merge_caller_body_tags(request_data=tag_target)
|
||||
LiteLLMProxyRequestSetup.merge_caller_body_tags(
|
||||
request_data=tag_target,
|
||||
metadata_variable_name=metadata_variable_name_for_request(route, tag_target),
|
||||
)
|
||||
LiteLLMProxyRequestSetup.apply_key_tags_pre_auth(request_data=tag_target, user_api_key_dict=valid_token)
|
||||
LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, request_body)
|
||||
|
||||
|
|
|
|||
|
|
@ -137,19 +137,34 @@ _CAPTURED_IDENTITY_CALL_TYPES: Final[frozenset[str]] = frozenset(
|
|||
_OBJECT_MAPPING: Final = TypeAdapter(ObjectMapping)
|
||||
|
||||
|
||||
def _proxy_stamped_used_client_oauth_token(
|
||||
request_data: Mapping[str, object], request_route: str | None
|
||||
) -> bool | None:
|
||||
def _proxy_stamped_metadata(request_data: Mapping[str, object], request_route: str | None) -> Mapping[str, object]:
|
||||
proxy_bucket: Final = (
|
||||
get_metadata_variable_name_from_kwargs(request_data)
|
||||
if request_route is None
|
||||
else metadata_variable_name_for_request(request_route, request_data)
|
||||
)
|
||||
proxy_metadata: Final = request_data.get(proxy_bucket)
|
||||
stamped: Final = proxy_metadata.get("used_client_oauth_token") if isinstance(proxy_metadata, dict) else None
|
||||
return _str_keyed_fields(request_data.get(proxy_bucket))
|
||||
|
||||
|
||||
def _proxy_stamped_used_client_oauth_token(
|
||||
request_data: Mapping[str, object], request_route: str | None
|
||||
) -> bool | None:
|
||||
stamped: Final = _proxy_stamped_metadata(request_data, request_route).get("used_client_oauth_token")
|
||||
return stamped if isinstance(stamped, bool) else None
|
||||
|
||||
|
||||
_PROXY_STAMPED_REQUEST_FIELDS: Final = ("requester_ip_address", "user_agent")
|
||||
|
||||
|
||||
def _proxy_stamped_request_fields(
|
||||
request_data: Mapping[str, object], request_route: str | None
|
||||
) -> Mapping[str, object]:
|
||||
if request_route is None:
|
||||
return {}
|
||||
proxy_metadata: Final = _proxy_stamped_metadata(request_data, request_route)
|
||||
return {field: proxy_metadata[field] for field in _PROXY_STAMPED_REQUEST_FIELDS if field in proxy_metadata}
|
||||
|
||||
|
||||
def _str_keyed_fields(value: object) -> Mapping[str, object]:
|
||||
return value if is_str_keyed_mapping(value) else {}
|
||||
|
||||
|
|
@ -455,7 +470,13 @@ class _ProxyDBLogger(CustomLogger):
|
|||
_metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_fields, request_route)
|
||||
|
||||
existing_metadata: Final[dict] = request_data.get("metadata", None) or {}
|
||||
existing_metadata.update({**_routed_metadata_fallback(request_fields), **_metadata})
|
||||
existing_metadata.update(
|
||||
{
|
||||
**_routed_metadata_fallback(request_fields),
|
||||
**_proxy_stamped_request_fields(request_fields, request_route),
|
||||
**_metadata,
|
||||
}
|
||||
)
|
||||
|
||||
litellm_metadata_bucket: Final = request_data.get("litellm_metadata")
|
||||
litellm_metadata: Final[ObjectMapping] = (
|
||||
|
|
|
|||
|
|
@ -655,6 +655,12 @@ def normalized_metadata_slot(
|
|||
return normalized
|
||||
|
||||
|
||||
def _holds_metadata_object(value: object) -> bool:
|
||||
if value is None or isinstance(value, dict):
|
||||
return True
|
||||
return isinstance(value, str) and isinstance(_DECODED_JSON.validate_python(safe_json_loads(value)), dict)
|
||||
|
||||
|
||||
def _metadata_object_or_error(field: str, value: object) -> Mapping[str, object] | ProxyException:
|
||||
if value is None:
|
||||
return {}
|
||||
|
|
@ -2429,7 +2435,7 @@ class LiteLLMProxyRequestSetup:
|
|||
if metadata_variable_name is not None
|
||||
else get_metadata_variable_name_from_kwargs(request_data)
|
||||
)
|
||||
if slot == "metadata":
|
||||
if slot == "metadata" or not _holds_metadata_object(request_data.get(slot)):
|
||||
return
|
||||
raw_caller_metadata: Final = request_data.get("metadata")
|
||||
caller_metadata: Final = (
|
||||
|
|
|
|||
|
|
@ -3699,6 +3699,20 @@ def test_get_guardrail_from_metadata() -> None:
|
|||
) == [{"test-guardrail": {"extra_body": {"key": "value"}}}]
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"caller_metadata",
|
||||
[{"guardrails": []}, {"guardrails": ["not-a-guardrail"]}, {"guardrails": None}],
|
||||
)
|
||||
def test_caller_metadata_guardrails_never_hide_the_proxy_guardrails_in_litellm_metadata(caller_metadata: dict) -> None:
|
||||
guardrail: Final = CustomGuardrail(
|
||||
guardrail_name="key-guardrail", event_hook=GuardrailEventHooks.pre_call, default_on=False
|
||||
)
|
||||
request_data: Final = {"metadata": caller_metadata, "litellm_metadata": {"guardrails": ["key-guardrail"]}}
|
||||
|
||||
assert guardrail.should_run_guardrail(request_data, GuardrailEventHooks.pre_call) is True
|
||||
|
||||
|
||||
def test_guardrail_is_in_requested_guardrails() -> None:
|
||||
guardrail: Final = CustomGuardrail(guardrail_name="test-guardrail")
|
||||
|
||||
|
|
|
|||
|
|
@ -623,6 +623,56 @@ async def test_async_post_call_failure_hook_reads_used_client_oauth_token_from_t
|
|||
assert json.loads(payload["metadata"])["used_client_oauth_token"] is expected
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"request_route, body",
|
||||
[
|
||||
(
|
||||
"/cursor/chat/completions",
|
||||
{
|
||||
"input": "Hello",
|
||||
"metadata": {"requester_ip_address": "203.0.113.9", "user_agent": "spoofed"},
|
||||
"litellm_metadata": {"requester_ip_address": "127.0.0.1", "user_agent": "Cursor/1.0"},
|
||||
},
|
||||
),
|
||||
(
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"metadata": {"requester_ip_address": "127.0.0.1", "user_agent": "Cursor/1.0"},
|
||||
"litellm_metadata": {"requester_ip_address": "203.0.113.9", "user_agent": "spoofed"},
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_async_post_call_failure_hook_logs_the_proxy_stamped_ip_and_user_agent(request_route: str, body: dict):
|
||||
request_data = {
|
||||
"model": "gpt-4o",
|
||||
"proxy_server_request": {"request_id": "test_request_id"},
|
||||
**body,
|
||||
}
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_update_database:
|
||||
await ProxyDBLogger().async_post_call_failure_hook(
|
||||
request_data=request_data,
|
||||
original_exception=Exception("invalid api key"),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key", request_route=request_route),
|
||||
)
|
||||
|
||||
payload = get_logging_payload(
|
||||
kwargs=mock_update_database.call_args[1]["kwargs"],
|
||||
response_obj={},
|
||||
start_time=datetime.now(),
|
||||
end_time=datetime.now(),
|
||||
)
|
||||
assert payload["requester_ip_address"] == "127.0.0.1"
|
||||
assert json.loads(payload["metadata"])["user_agent"] == "Cursor/1.0"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_post_call_failure_hook_bills_guardrail_cost_on_blocked_request():
|
||||
"""LIT-5651: a request blocked by a guardrail never reaches the LLM, but the
|
||||
|
|
|
|||
|
|
@ -698,6 +698,45 @@ async def test_add_litellm_data_to_request_rejects_non_object_metadata(field, va
|
|||
assert field not in data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("route", ["/v1/chat/completions", "/v1/responses", "/cursor/chat/completions"])
|
||||
@pytest.mark.parametrize("value, received_type", [("junk", "a string"), (5, "an integer"), ([1], "an array")])
|
||||
async def test_invalid_litellm_metadata_answers_400_when_metadata_carries_tags(route, value, received_type):
|
||||
from litellm.proxy.auth.auth_checks import common_checks
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
data = {"model": "gpt-4o", "input": "hi", "metadata": {"tags": ["client-tag"]}, "litellm_metadata": value}
|
||||
request = _batches_request_mock()
|
||||
await common_checks(
|
||||
request_body=data,
|
||||
team_object=None,
|
||||
user_object=None,
|
||||
end_user_object=None,
|
||||
global_proxy_spend=None,
|
||||
general_settings={},
|
||||
route=route,
|
||||
llm_router=None,
|
||||
proxy_logging_obj=ProxyLogging(user_api_key_cache=None),
|
||||
valid_token=UserAPIKeyAuth(api_key="hashed-key"),
|
||||
request=request,
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await add_litellm_data_to_request(
|
||||
data=data,
|
||||
request=request,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
|
||||
proxy_config=MagicMock(),
|
||||
general_settings={},
|
||||
version="test-version",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "400"
|
||||
assert exc_info.value.message == (
|
||||
f"Invalid type for 'litellm_metadata': expected an object, but got {received_type} instead."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_litellm_data_to_request_removes_every_invalid_metadata_field_before_raising():
|
||||
"""When both fields are invalid, the raise for the first must not leave the
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue