This commit is contained in:
Joshua Hoblitt 2026-10-27 02:24:25 +00:00 • committed by GitHub
commit 83198e013a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 140 additions and 17 deletions

View file

@ -262,6 +262,9 @@ class GenericGuardrailAPI(CustomGuardrail):
"blocked_reason": str (optional, only if action is BLOCKED),
"text": str (optional, modified text if action is GUARDRAIL_INTERVENED)
}
A non-empty but malformed texts/images/tools/structured_messages field fails the call closed;
with fail_on_error=false the original content passes through instead. Empty/falsy values are ignored.
"""
def __init__(

View file

@ -1,8 +1,8 @@
from collections.abc import Mapping, Sequence
from typing import Any, Final, Literal, cast # noqa: TID251 # JSON chat rows have no typed constructor across roles
from pydantic import ConfigDict, Field, SerializerFunctionWrapHandler, model_serializer
from typing_extensions import TypedDict
from pydantic import ConfigDict, Field, SerializerFunctionWrapHandler, TypeAdapter, ValidationError, model_serializer
from typing_extensions import TypedDict, TypeVar
from litellm.types.guardrails import GuardrailNumRetries
from litellm.types.llms.base import LiteLLMBaseModel
@ -180,14 +180,35 @@ def coerce_stream_holdback_value(value: Any) -> int:
return 0
def _is_placeholder(value: object) -> bool:
return not value and not isinstance(value, list)
def structured_messages_from_response(value: object) -> Sequence[AllMessageValues] | None:
if not isinstance(value, list):
return None
if not all(isinstance(message, Mapping) and isinstance(message.get("role"), str) for message in value):
if _is_placeholder(value):
return None
if not isinstance(value, list) or not all(
isinstance(message, Mapping) and isinstance(message.get("role"), str) for message in value
):
raise ValueError("Generic Guardrail API response field 'structured_messages' is malformed")
return cast("Sequence[AllMessageValues]", value) # cast-ok: JSON rows checked for a role, the same trust texts get
_Parsed: Final = TypeVar("_Parsed")
_STRINGS: Final = TypeAdapter(list[str])
_JSON_ROWS: Final = TypeAdapter(list[dict[str, object]])
def _parsed_field(adapter: TypeAdapter[_Parsed], data: Mapping[str, object], field: str) -> _Parsed | None:
value: Final = data.get(field)
if _is_placeholder(value):
return None
try:
return adapter.validate_python(value, strict=True)
except ValidationError as error:
raise ValueError(f"Generic Guardrail API response field '{field}' is malformed") from error
class GenericGuardrailAPIResponse:
"""Response model for the Generic Guardrail API"""
@ -229,9 +250,11 @@ class GenericGuardrailAPIResponse:
return cls(
action=data.get("action", "NONE"),
blocked_reason=data.get("blocked_reason"),
texts=data.get("texts"),
images=data.get("images"),
tools=data.get("tools"),
texts=_parsed_field(_STRINGS, data, "texts"),
images=_parsed_field(_STRINGS, data, "images"),
tools=cast( # cast-ok: tool rows pass through as the guardrail returned them, as before
"list[GuardrailToolParam] | None", _parsed_field(_JSON_ROWS, data, "tools")
),
stream_holdback_chars=stream_holdback_chars,
structured_messages=structured_messages_from_response(data.get("structured_messages")),
)

View file

@ -8,6 +8,7 @@ specifically focusing on metadata extraction and passing.
import json
import logging
import os
import re
from collections.abc import Callable
from functools import partial
from queue import SimpleQueue
@ -982,19 +983,12 @@ class TestStructuredMessagesInResponse:
assert guardrailed_inputs["structured_messages"] == [returned_row]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"structured_messages",
[[], [{"content": "a row with no role"}], "not a list"],
ids=["empty", "no_role", "not_a_list"],
)
async def test_rows_that_are_not_chat_messages_are_ignored(
self, generic_guardrail, mock_request_data_input, structured_messages
):
async def test_empty_structured_messages_are_ignored(self, generic_guardrail, mock_request_data_input):
mock_response = MagicMock()
mock_response.json.return_value = {
"action": "GUARDRAIL_INTERVENED",
"texts": ["[REDACTED]"],
"structured_messages": structured_messages,
"structured_messages": [],
}
mock_response.raise_for_status = MagicMock()
@ -2621,3 +2615,106 @@ class TestNumRetries:
assert sent.qsize() == num_retries + 1
assert result == {"texts": ["hi"]}
def _answering(body: dict[str, object]) -> MagicMock:
answer: Final = MagicMock()
answer.json.return_value = body
answer.raise_for_status = MagicMock()
return answer
class TestMalformedResponseFieldsFailClosed:
@pytest.mark.asyncio
@pytest.mark.parametrize(
("field", "value"),
[
("texts", "x [R] y"),
("texts", ["x [R] y", 5]),
("images", "https://example.com/a.png"),
("images", [5]),
("tools", "lookup"),
("tools", ["lookup"]),
("structured_messages", "x [R] y"),
("structured_messages", [{"content": "x [R] y"}]),
],
)
async def test_malformed_field_fails_the_call_instead_of_passing_content_through(
self, generic_guardrail: GenericGuardrailAPI, field: str, value: object
):
response: Final = _answering({"action": "GUARDRAIL_INTERVENED", field: value})
with patch.object(generic_guardrail.async_handler, "post", return_value=response):
with pytest.raises(
Exception,
match=re.escape(
f"Generic Guardrail API failed: Generic Guardrail API response field '{field}' is malformed"
),
):
await generic_guardrail.apply_guardrail(
inputs={"texts": ["x secret y"]}, request_data={}, input_type="request"
)
@pytest.mark.asyncio
async def test_malformed_field_passes_content_through_when_fail_on_error_is_false(self):
guardrail: Final = GenericGuardrailAPI(
api_base="https://api.test.guardrail.com",
guardrail_name="test-fail-open-guardrail",
event_hook="pre_call",
default_on=True,
fail_on_error=False,
)
response: Final = _answering({"action": "GUARDRAIL_INTERVENED", "texts": "x [R] y"})
with patch.object(guardrail.async_handler, "post", return_value=response):
returned: Final = await guardrail.apply_guardrail(
inputs={"texts": ["x secret y"]}, request_data={}, input_type="request"
)
assert returned == {"texts": ["x secret y"]}
@pytest.mark.asyncio
async def test_well_formed_fields_are_applied(self, generic_guardrail: GenericGuardrailAPI):
rows: Final = [{"role": "user", "content": "x [R] y"}]
response: Final = _answering(
{"action": "GUARDRAIL_INTERVENED", "texts": ["x [R] y"], "structured_messages": rows}
)
with patch.object(generic_guardrail.async_handler, "post", return_value=response):
returned: Final = await generic_guardrail.apply_guardrail(
inputs={"texts": ["x secret y"], "structured_messages": [{"role": "user", "content": "x secret y"}]},
request_data={},
input_type="request",
)
assert (returned["texts"], returned.get("structured_messages")) == (["x [R] y"], rows)
@pytest.mark.asyncio
@pytest.mark.parametrize("action", ["NONE", "GUARDRAIL_INTERVENED"])
@pytest.mark.parametrize(
"placeholder", [None, "", {}, False, 0], ids=["null", "empty_str", "empty_dict", "false", "zero"]
)
async def test_empty_placeholder_fields_are_treated_as_absent(
self, generic_guardrail: GenericGuardrailAPI, action: str, placeholder: object
):
response: Final = _answering(
{
"action": action,
"texts": placeholder,
"images": placeholder,
"tools": placeholder,
"structured_messages": placeholder,
}
)
inputs: Final[GenericGuardrailAPIInputs] = {
"texts": ["x secret y"],
"images": ["https://example.com/a.png"],
"tools": [{"type": "function", "function": {"name": "lookup"}}],
}
with patch.object(generic_guardrail.async_handler, "post", return_value=response):
returned: Final = await generic_guardrail.apply_guardrail(
inputs=inputs, request_data={}, input_type="request"
)
assert returned == inputs