mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-12 08:42:18 +00:00
fix(proxy): scan a pass-through reply's text, not its JSON envelope, in post-call guardrails
A post-call guardrail on a route with no typed call type read the serialized response, so a unix timestamp or an embedding float tripped a phone or card pattern and blocked a clean reply. Without response_fields it now reads the strings the reply carries, minus ids and encoded data.
This commit is contained in:
parent
03290d5902
commit
0aaa521e15
4 changed files with 119 additions and 5 deletions
|
|
@ -24,7 +24,7 @@ litellm/proxy/pass_through_endpoints/
|
|||
The `PassThroughEndpointHandler` enables guardrails to run on passthrough endpoint requests by:
|
||||
|
||||
1. **Field Targeting**: Extracts specific fields from the request/response using JSONPath expressions configured in `request_fields` / `response_fields`
|
||||
2. **Full Payload Fallback**: If no field targeting is configured, processes the entire payload
|
||||
2. **Full Payload Fallback**: If no field targeting is configured, processes the entire request payload, or the text fields of the response
|
||||
3. **Config Access**: Uses `get_passthrough_guardrails_config()` / `set_passthrough_guardrails_config()` helpers to access the passthrough guardrails configuration stored in request metadata
|
||||
|
||||
## Example Config
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from pydantic import TypeAdapter
|
|||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_guardrail import without_server_streaming_classification
|
||||
from litellm.litellm_core_utils.core_helpers import has_attributes
|
||||
from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, has_attributes
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import (
|
||||
BaseTranslation,
|
||||
StreamingScanKey,
|
||||
|
|
@ -29,6 +29,47 @@ if TYPE_CHECKING:
|
|||
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
|
||||
|
||||
# Strings in a provider's JSON response that name or encode something rather than carry the reply's text.
|
||||
_NON_TEXT_RESPONSE_FIELDS: Final = frozenset(
|
||||
(
|
||||
"b64_json",
|
||||
"bytesBase64Encoded",
|
||||
"embedding",
|
||||
"finishReason",
|
||||
"finish_reason",
|
||||
"id",
|
||||
"model",
|
||||
"modelVersion",
|
||||
"object",
|
||||
"responseId",
|
||||
"role",
|
||||
"service_tier",
|
||||
"status",
|
||||
"system_fingerprint",
|
||||
"thoughtSignature",
|
||||
"type",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _response_text(response: Mapping[str, object]) -> str:
|
||||
"""The text a JSON response carries, without its timestamps, usage counts, ids and vectors, which a guardrail
|
||||
would otherwise read as phone or card numbers."""
|
||||
pending: Final[list[object]] = [response] # mutable-ok: local to this walk
|
||||
texts: Final[list[str]] = [] # mutable-ok: local to this walk
|
||||
while pending:
|
||||
value = pending.pop()
|
||||
fields = as_str_mapping(value)
|
||||
items = as_object_list(value)
|
||||
if isinstance(value, str):
|
||||
texts.append(value)
|
||||
elif fields is not None:
|
||||
pending.extend(reversed([item for name, item in fields.items() if name not in _NON_TEXT_RESPONSE_FIELDS]))
|
||||
elif items is not None:
|
||||
pending.extend(reversed(items))
|
||||
return "\n".join(text for text in texts if text)
|
||||
|
||||
|
||||
class PassThroughEndpointHandler(BaseTranslation):
|
||||
"""
|
||||
Handler for processing passthrough endpoint requests with guardrails.
|
||||
|
|
@ -153,6 +194,7 @@ class PassThroughEndpointHandler(BaseTranslation):
|
|||
litellm_logging_obj: Optional logging object
|
||||
user_api_key_dict: User API key metadata to pass to guardrails
|
||||
"""
|
||||
response_fields: Final = as_str_mapping(response)
|
||||
if not isinstance(response, dict):
|
||||
verbose_proxy_logger.debug("PassThroughEndpointHandler: Response is not a dict, skipping")
|
||||
return response
|
||||
|
|
@ -168,7 +210,11 @@ class PassThroughEndpointHandler(BaseTranslation):
|
|||
field_expressions: Final = settings.response_fields if settings else None
|
||||
|
||||
# Extract text to check
|
||||
text_to_check: Final = self._extract_text_for_guardrail(response, field_expressions)
|
||||
text_to_check: Final = (
|
||||
self._extract_text_for_guardrail(response, field_expressions)
|
||||
if field_expressions
|
||||
else _response_text(response_fields or {})
|
||||
)
|
||||
|
||||
if not text_to_check:
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ OPENAI_ROUTE: Final = "/openai/v1/chat/completions"
|
|||
ANTHROPIC_ROUTE: Final = "/anthropic/v1/messages"
|
||||
GEMINI_ROUTE: Final = "/gemini/v1beta/models/gemini-2.5-flash:generateContent"
|
||||
CUSTOM_ROUTE: Final = "/passthrough-auth-custom"
|
||||
PHONE_POST_CALL: Final = "passthrough-auth-phone-post-call"
|
||||
_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
|
||||
_JSON: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_JSON_LIST: Final = TypeAdapter(list[JsonValue])
|
||||
|
|
@ -66,7 +67,11 @@ def _verdict(request: Request) -> Reply:
|
|||
|
||||
|
||||
_OPENAI_BODY: Final = json.dumps(
|
||||
_JSON.validate_json(chat_reply("chatcmpl-passthrough-auth", "gpt-5.6", "Hi", stream=False).body), indent=2
|
||||
{
|
||||
**_JSON.validate_json(chat_reply("chatcmpl-passthrough-auth", "gpt-5.6", "Hi", stream=False).body),
|
||||
"created": 1793600123,
|
||||
},
|
||||
indent=2,
|
||||
).encode()
|
||||
|
||||
|
||||
|
|
@ -222,6 +227,17 @@ def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> I
|
|||
"api_key": "synthetic-guardrail-key",
|
||||
},
|
||||
},
|
||||
{
|
||||
"guardrail_name": PHONE_POST_CALL,
|
||||
"litellm_params": {
|
||||
"guardrail": "litellm_content_filter",
|
||||
"mode": "post_call",
|
||||
"default_on": False,
|
||||
"patterns": [
|
||||
{"pattern_type": "prebuilt", "pattern_name": "us_phone", "action": "BLOCK"}
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
"guardrail_name": MODEL_POST_CALL,
|
||||
"litellm_params": {
|
||||
|
|
@ -415,6 +431,18 @@ def test_key_apply_guardrail_post_call_scans_the_provider_reply(rig: AuthGuardra
|
|||
assert [b'"Hi' in request.body for request in rig.generic_post_call_vendor.drain()] == [True]
|
||||
|
||||
|
||||
def test_post_call_guardrail_does_not_read_a_reply_timestamp_as_a_phone_number(rig: AuthGuardrailRig) -> None:
|
||||
gateway: Final = rig.owned.gateway
|
||||
with gateway.scenario() as scenario:
|
||||
key: Final = scenario.key(metadata={"guardrails": [PHONE_POST_CALL]})
|
||||
_drain(rig)
|
||||
|
||||
response: Final = _send(gateway, OPENAI_ROUTE, _route_body(OPENAI_ROUTE, "What is the capital of France?"), key)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["created"] == 1793600123
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", [OPENAI_ROUTE, ANTHROPIC_ROUTE, GEMINI_ROUTE])
|
||||
def test_guardrail_the_client_names_in_metadata_still_runs_for_a_guarded_key(rig: AuthGuardrailRig, route: str) -> None:
|
||||
gateway: Final = rig.owned.gateway
|
||||
|
|
@ -539,7 +567,7 @@ def _charged_rows(gateway: Gateway, key: str) -> tuple[Mapping[str, object], ...
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", [OPENAI_ROUTE, ANTHROPIC_ROUTE, GEMINI_ROUTE])
|
||||
@pytest.mark.parametrize("route", [ANTHROPIC_ROUTE, GEMINI_ROUTE])
|
||||
def test_response_a_post_call_guardrail_blocks_still_charges_the_provider_call(
|
||||
rig: AuthGuardrailRig, route: str
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import json
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
|
@ -226,3 +227,42 @@ def test_passthrough_route_block_before_any_invoke_frame_is_a_whole_message_in_c
|
|||
"message_stop",
|
||||
]
|
||||
assert events[2]["delta"]["text"] == "blocked"
|
||||
|
||||
|
||||
async def _scanned_response_texts(response: dict[str, object]) -> tuple[str, ...]:
|
||||
guardrail: Final = MagicMock(guardrail_name="post-call", apply_guardrail=AsyncMock())
|
||||
await PassThroughEndpointHandler().process_output_response(response, guardrail, request_data={})
|
||||
return tuple(text for call in guardrail.apply_guardrail.await_args_list for text in call.kwargs["inputs"]["texts"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_scan_reads_the_reply_text_and_not_the_numbers_around_it():
|
||||
response: Final = {
|
||||
"id": "chatcmpl-1793600123",
|
||||
"object": "chat.completion",
|
||||
"created": 1793600123,
|
||||
"model": "gpt-5.6",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Paris."}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 4111111111111111, "total_tokens": 5551234567},
|
||||
}
|
||||
|
||||
assert await _scanned_response_texts(response) == ("Paris.",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_scan_still_reads_every_string_a_reply_carries():
|
||||
response: Final = {
|
||||
"choices": [
|
||||
{"message": {"content": "call 555-123-4567", "tool_calls": [{"function": {"arguments": '{"a": "b"}'}}]}}
|
||||
],
|
||||
"candidates": [{"content": {"parts": [{"text": "card 4111 1111 1111 1111"}]}}],
|
||||
}
|
||||
|
||||
assert await _scanned_response_texts(response) == ('call 555-123-4567\n{"a": "b"}\ncard 4111 1111 1111 1111',)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_scan_skips_a_reply_with_no_text():
|
||||
response: Final = {"object": "list", "data": [{"object": "embedding", "embedding": [0.4111111111111111]}]}
|
||||
|
||||
assert await _scanned_response_texts(response) == ()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue