From c4c07866d830f6e14aad24440c56d668390d4526 Mon Sep 17 00:00:00 2001 From: Caduri Katzav Date: Thu, 1 Oct 2026 11:58:02 +0300 Subject: [PATCH] fix(guardrails): send rejected-part rows as the caller wrote them A row whose part list holds a part the request model rejects is now posted with the caller's content as is. The as_json_value round trip it went through is gone: the whole request is already dumped by pydantic before that point, so it never kept extra nesting depth, and no test needed it The two JSON builds in the row helpers carry the same mutable-ok as the POST body, and a new test covers /v1/responses input_text rows, which the fix already handled --- litellm/proxy/guardrails/_content_utils.py | 13 ---------- .../generic_guardrail_api.py | 17 ++++++------- .../test_generic_guardrail_api.py | 24 +++++++++++++++++-- .../proxy/guardrails/test_content_utils.py | 12 ---------- 4 files changed, 31 insertions(+), 35 deletions(-) diff --git a/litellm/proxy/guardrails/_content_utils.py b/litellm/proxy/guardrails/_content_utils.py index 4f0a94f7f88..2a2ef5217b8 100644 --- a/litellm/proxy/guardrails/_content_utils.py +++ b/litellm/proxy/guardrails/_content_utils.py @@ -8,13 +8,9 @@ skip the other shapes — these helpers normalise that so every hook sees every text fragment. """ -import json from collections.abc import Callable, Iterator, Mapping, Sequence from typing import Any, Final -from pydantic import JsonValue -from pydantic_core import to_jsonable_python - # Call types whose body carries free-form chat / prompt text that # text-content guardrails (banned keywords, content moderation, secret # detection, …) should inspect. The proxy ingress passes ``route_type`` @@ -311,12 +307,3 @@ def build_inspection_messages(data: dict[str, Any]) -> list[dict[str, str]]: role = message.get("role", "user") or "user" flattened.append({"role": role, "content": text}) return flattened - - -def as_json_value(value: object) -> JsonValue: - """Round-trips through the stdlib codec because pydantic's serializer turns anything nested - past 254 levels into "...", while this keeps about the depth the proxy's request parser accepts""" - parsed: Final[JsonValue] = json.loads( # pyright: ignore[reportAny] # untyped stdlib parse of json.dumps output - json.dumps(value, default=to_jsonable_python) - ) - return parsed diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index 7b59d509a5b..0b6b9dcaecd 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -26,7 +26,6 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.proxy.guardrails._content_utils import as_json_value from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( @@ -159,22 +158,24 @@ def _is_part_list(value: object) -> TypeIs[list[object]]: # guard-ok: trivial i return isinstance(value, list) -def _row_as_sent(dumped: JsonValue, caller: Mapping[str, object]) -> JsonValue: - """The request model validates list content lazily and dumps a list holding - any part it rejects as [], so such a row is sent with the caller's content.""" +def _row_as_sent(dumped: JsonValue, caller: Mapping[str, object]) -> object: + """The request model dumps a part list holding any part it rejects as [], so such a row is sent with the + caller's content""" caller_content: Final = caller.get("content") if not isinstance(dumped, dict) or not _is_part_list(caller_content): return dumped dumped_content: Final = dumped.get("content") if isinstance(dumped_content, list) and len(dumped_content) == len(caller_content): return dumped - return {**dumped, "content": as_json_value(caller_content)} + return {**dumped, "content": caller_content} # mutable-ok: json.dumps encodes dict, not MappingProxyType -def _rows_as_sent(dumped_rows: JsonValue, caller_rows: Sequence[Mapping[str, object]] | None) -> JsonValue: +def _rows_as_sent(dumped_rows: JsonValue, caller_rows: Sequence[Mapping[str, object]] | None) -> object: if caller_rows is None or not isinstance(dumped_rows, list): return dumped_rows - return [_row_as_sent(dumped, caller) for dumped, caller in zip(dumped_rows, caller_rows, strict=True)] + return [ # mutable-ok: a JSON array, which structured_messages_from_json requires to be a list + _row_as_sent(dumped, caller) for dumped, caller in zip(dumped_rows, caller_rows, strict=True) + ] def _structured_rows_to_write_back( @@ -503,7 +504,7 @@ class GenericGuardrailAPI(CustomGuardrail): # The model's list content is a lazy iterator that this dump consumes, so it cannot be read again dumped: Final[Mapping[str, JsonValue]] = guardrail_request.model_dump(mode="json") sent_messages: Final = _rows_as_sent(dumped.get("structured_messages"), structured_messages) - request_json: Final = {**dumped, "structured_messages": sent_messages} # mutable-ok: JSON POST body + request_json: Final = {**dumped, "structured_messages": sent_messages} # mutable-ok: post() needs a dict response: Final = await self.async_handler.post( url=self.api_base, diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py index 3af84c760be..45657d45b06 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py @@ -8,7 +8,7 @@ specifically focusing on metadata extraction and passing. import json import os from collections.abc import Callable, Mapping -from typing import Final +from typing import Final, TypeAlias from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -22,6 +22,7 @@ from litellm.exceptions import GuardrailRaisedException, Timeout from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.openai.chat.guardrail_translation.handler import OpenAIChatCompletionsHandler +from litellm.llms.openai.responses.guardrail_translation.handler import OpenAIResponsesHandler from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( GenericGuardrailAPI, @@ -737,7 +738,7 @@ def _image_part() -> ChatCompletionImageObject: return {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}} -GuardrailAnswer = Callable[[Mapping[str, JsonValue]], Mapping[str, JsonValue]] +GuardrailAnswer: TypeAlias = Callable[[Mapping[str, JsonValue]], Mapping[str, JsonValue]] def _guardrail_answering(answer: GuardrailAnswer) -> GenericGuardrailAPI: @@ -833,6 +834,13 @@ async def _llm_bound_anthropic_messages( return data["messages"] +async def _llm_bound_responses_input(guardrail: GenericGuardrailAPI, input_items: list[JsonValue]) -> object: + data: Final = await OpenAIResponsesHandler().process_input_messages( + data={"model": "gpt-5.6", "input": input_items}, guardrail_to_apply=guardrail + ) + return data["input"] + + class TestEchoedRowsReachingTheLLM: @pytest.mark.asyncio @pytest.mark.parametrize( @@ -888,6 +896,18 @@ class TestEchoedRowsReachingTheLLM: {"role": "user", "content": [{"type": "text", "text": "my ssn is [SSN]"}, {"type": "text", "text": "ok"}]} ], "an unchanged echo of every content block row must leave the rewrite to texts" + @pytest.mark.asyncio + async def test_every_responses_input_text_row_echoed_applies_the_masked_texts(self) -> None: + guardrail: Final = _guardrail_answering(_echo_every_row_and_mask_texts) + + llm_bound: Final = await _llm_bound_responses_input( + guardrail, [{"role": "user", "content": [{"type": "input_text", "text": f"my ssn is {_SSN}"}]}] + ) + + assert llm_bound == [{"role": "user", "content": [{"type": "input_text", "text": "my ssn is [SSN]"}]}], ( + "an unchanged echo of every input_text row must leave the rewrite to texts" + ) + @pytest.mark.asyncio async def test_an_echoed_multipart_row_is_restored_to_the_callers_row(self) -> None: guardrail: Final = _guardrail_answering(_echo_first_row_and_mask_the_rest) diff --git a/tests/unit/proxy/guardrails/test_content_utils.py b/tests/unit/proxy/guardrails/test_content_utils.py index 527e0f39c94..920ffc77095 100644 --- a/tests/unit/proxy/guardrails/test_content_utils.py +++ b/tests/unit/proxy/guardrails/test_content_utils.py @@ -2,7 +2,6 @@ from litellm.proxy.guardrails._content_utils import ( apply_redacted_messages_back, - as_json_value, build_inspection_messages, has_non_string_content, is_non_conversational_call_type, @@ -742,14 +741,3 @@ def test_is_non_conversational_call_type_defaults_to_inspecting_unknown_call_typ """A call type this module has never heard of must still be inspected — failing closed is the point of the deny-list.""" assert is_non_conversational_call_type("some_future_call_type") is False - - -def _nested(depth: int) -> dict[str, object]: - return {"leaf": "x"} if depth == 0 else {"nested": _nested(depth - 1)} - - -def test_as_json_value_keeps_content_nested_past_the_pydantic_serializer_limit_and_decodes_bytes(): - assert as_json_value([{"type": "document", "source": _nested(600), "data": b"raw"}, ("a", 1)]) == [ - {"type": "document", "source": _nested(600), "data": "raw"}, - ["a", 1], - ], "nothing may be truncated to '...' and non-JSON types must become their JSON form"