From a708475f28bafd22bc6fa180709b53f8be8ff382 Mon Sep 17 00:00:00 2001 From: L4XB Date: Mon, 14 Sep 2026 22:48:34 +0200 Subject: [PATCH 1/3] fix(aporia): only `passthrough` may go out unchanged MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aporia answers with one of four actions, which the line's own comment names. The hook acted on `block` and forwarded the request unchanged for the other three. `modify` and `rephrase` are interventions — Aporia is saying the content should not ship as written — and litellm forwarded the original with a 200 and no log line, so an ignored intervention and a clean scan were indistinguishable downstream. An unrecognised or missing action forwarded too, because `.get("action")` has no default: a response shape change, or an error object without that key, read as permission. Only `passthrough` now proceeds. litellm cannot apply a `modify` or a `rephrase` from this response, so refusing is the honest answer for both. Fixes #41097 --- .../guardrail_hooks/aporia_ai/aporia_ai.py | 9 +- .../guardrails/guardrail_hooks/test_aporia.py | 93 +++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aporia.py diff --git a/litellm/proxy/guardrails/guardrail_hooks/aporia_ai/aporia_ai.py b/litellm/proxy/guardrails/guardrail_hooks/aporia_ai/aporia_ai.py index dafa6e06652..cd6b0e26385 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/aporia_ai/aporia_ai.py +++ b/litellm/proxy/guardrails/guardrail_hooks/aporia_ai/aporia_ai.py @@ -129,7 +129,14 @@ class AporiaGuardrail(CustomGuardrail): # check if the response was flagged _json_response: Final = response.json() action: str = _json_response.get("action") # possible values are modify, passthrough, block, rephrase - if action == "block": + # Only `passthrough` says the content may go out as written. + # `modify` and `rephrase` are interventions: Aporia is saying this + # should not ship unchanged, and litellm cannot apply either from + # this response, so forwarding the original would defeat the + # guardrail while reporting success. An unrecognised or missing + # action is treated the same way: a guardrail that cannot tell what + # it was told must not be the one to decide the content is fine. + if action != "passthrough": raise HTTPException( status_code=400, detail={ diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aporia.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aporia.py new file mode 100644 index 00000000000..1c7177e0386 --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aporia.py @@ -0,0 +1,93 @@ +"""Tests for how the Aporia guardrail acts on Aporia's verdict. + +Aporia answers with one of four actions. Only ``passthrough`` says the content +may go out as written; ``modify`` and ``rephrase`` are interventions. Forwarding +the original for those defeats the guardrail and reports success while doing it +(#41097). +""" + +import json +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import HTTPException + +from litellm.proxy.guardrails.guardrail_hooks.aporia_ai.aporia_ai import AporiaGuardrail + + +def _guardrail(action: Any, include_action: bool = True) -> AporiaGuardrail: + """An AporiaGuardrail whose /validate call answers with one verdict.""" + guardrail = AporiaGuardrail( + guardrail_name="aporia", + api_key="k", + api_base="https://example.invalid", + ) + body = {"action": action} if include_action else {"reason": "no action key"} + response = MagicMock() + response.status_code = 200 + response.text = json.dumps(body) + response.json = MagicMock(return_value=body) + guardrail.async_handler = MagicMock() + guardrail.async_handler.post = AsyncMock(return_value=response) + return guardrail + + +async def _validate(guardrail: AporiaGuardrail) -> None: + await guardrail.make_aporia_api_request( + request_data={}, + new_messages=[{"role": "user", "content": "hello"}], + ) + + +@pytest.mark.asyncio +async def test_passthrough_is_forwarded(): + """The accept control: a cleanly scanned request must still go out, or the + guardrail is a wall rather than a filter.""" + guardrail = _guardrail("passthrough") + + await _validate(guardrail) + + # It reached Aporia and came back without raising, which is the whole + # observable effect of letting a request through. + guardrail.async_handler.post.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_block_is_refused(): + with pytest.raises(HTTPException) as exc: + await _validate(_guardrail("block")) + + assert exc.value.status_code == 400 + assert "Violated guardrail policy" in str(exc.value.detail) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ["modify", "rephrase"]) +async def test_an_intervention_is_not_forwarded_unchanged(action): + """Aporia is saying this should not ship as written, and litellm cannot + apply the modification from this response — so it must not ship it.""" + with pytest.raises(HTTPException) as exc: + await _validate(_guardrail(action)) + + assert exc.value.status_code == 400 + assert action in str(exc.value.detail) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "action,include_action", + [ + ("BLOCK", True), + ("blocked", True), + ("", True), + (None, True), + (None, False), + ], +) +async def test_a_verdict_it_cannot_read_is_not_taken_for_permission(action, include_action): + """A response shape change, or an error object with no action at all, used + to forward. A guardrail that cannot tell what it was told is not the one to + decide the content is fine.""" + with pytest.raises(HTTPException): + await _validate(_guardrail(action, include_action=include_action)) From 3105fded3f1cb1a2527df80ee6cf9eed985b05fd Mon Sep 17 00:00:00 2001 From: L4XB Date: Wed, 23 Sep 2026 20:37:51 +0200 Subject: [PATCH 2/3] fix(aporia): inject the HTTP handler and drop the explanatory comments The tests replaced the guardrail's async_handler attribute after construction. The constructor now takes an optional async_handler, and the tests pass a mocked one in The verdict check loses its comment block, and the test helper takes a typed response body instead of Any --- .../guardrail_hooks/aporia_ai/aporia_ai.py | 20 +++--- .../guardrails/guardrail_hooks/test_aporia.py | 71 +++++++------------ 2 files changed, 38 insertions(+), 53 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/aporia_ai/aporia_ai.py b/litellm/proxy/guardrails/guardrail_hooks/aporia_ai/aporia_ai.py index cd6b0e26385..21211da5fa5 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/aporia_ai/aporia_ai.py +++ b/litellm/proxy/guardrails/guardrail_hooks/aporia_ai/aporia_ai.py @@ -24,6 +24,7 @@ from litellm.litellm_core_utils.logging_utils import ( convert_litellm_response_object_to_str, ) from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, get_async_httpx_client, httpxSpecialProvider, ) @@ -44,9 +45,17 @@ class AporiaGuardrail(CustomGuardrail): GuardrailEventHooks.post_call, ] - def __init__(self, api_key: str | None = None, api_base: str | None = None, **kwargs): + def __init__( + self, + api_key: str | None = None, + api_base: str | None = None, + async_handler: AsyncHTTPHandler | None = None, + **kwargs, + ): kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) - self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) + self.async_handler = async_handler or get_async_httpx_client( + llm_provider=httpxSpecialProvider.GuardrailCallback + ) self.aporia_api_key = api_key or os.environ["APORIO_API_KEY"] self.aporia_api_base = api_base or os.environ["APORIO_API_BASE"] super().__init__(**kwargs) @@ -129,13 +138,6 @@ class AporiaGuardrail(CustomGuardrail): # check if the response was flagged _json_response: Final = response.json() action: str = _json_response.get("action") # possible values are modify, passthrough, block, rephrase - # Only `passthrough` says the content may go out as written. - # `modify` and `rephrase` are interventions: Aporia is saying this - # should not ship unchanged, and litellm cannot apply either from - # this response, so forwarding the original would defeat the - # guardrail while reporting success. An unrecognised or missing - # action is treated the same way: a guardrail that cannot tell what - # it was told must not be the one to decide the content is fine. if action != "passthrough": raise HTTPException( status_code=400, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aporia.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aporia.py index 1c7177e0386..58272acfa11 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aporia.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aporia.py @@ -1,36 +1,28 @@ -"""Tests for how the Aporia guardrail acts on Aporia's verdict. - -Aporia answers with one of four actions. Only ``passthrough`` says the content -may go out as written; ``modify`` and ``rephrase`` are interventions. Forwarding -the original for those defeats the guardrail and reports success while doing it -(#41097). -""" - import json -from typing import Any from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy.guardrails.guardrail_hooks.aporia_ai.aporia_ai import AporiaGuardrail -def _guardrail(action: Any, include_action: bool = True) -> AporiaGuardrail: - """An AporiaGuardrail whose /validate call answers with one verdict.""" - guardrail = AporiaGuardrail( - guardrail_name="aporia", - api_key="k", - api_base="https://example.invalid", - ) - body = {"action": action} if include_action else {"reason": "no action key"} +def _guardrail(body: dict[str, str | None]) -> tuple[AporiaGuardrail, AsyncMock]: response = MagicMock() response.status_code = 200 response.text = json.dumps(body) response.json = MagicMock(return_value=body) - guardrail.async_handler = MagicMock() - guardrail.async_handler.post = AsyncMock(return_value=response) - return guardrail + post = AsyncMock(return_value=response) + handler = MagicMock(spec=AsyncHTTPHandler) + handler.post = post + guardrail = AporiaGuardrail( + guardrail_name="aporia", + api_key="k", + api_base="https://example.invalid", + async_handler=handler, + ) + return guardrail, post async def _validate(guardrail: AporiaGuardrail) -> None: @@ -42,21 +34,19 @@ async def _validate(guardrail: AporiaGuardrail) -> None: @pytest.mark.asyncio async def test_passthrough_is_forwarded(): - """The accept control: a cleanly scanned request must still go out, or the - guardrail is a wall rather than a filter.""" - guardrail = _guardrail("passthrough") + guardrail, post = _guardrail({"action": "passthrough"}) await _validate(guardrail) - # It reached Aporia and came back without raising, which is the whole - # observable effect of letting a request through. - guardrail.async_handler.post.assert_awaited_once() + post.assert_awaited_once() @pytest.mark.asyncio async def test_block_is_refused(): + guardrail, _ = _guardrail({"action": "block"}) + with pytest.raises(HTTPException) as exc: - await _validate(_guardrail("block")) + await _validate(guardrail) assert exc.value.status_code == 400 assert "Violated guardrail policy" in str(exc.value.detail) @@ -64,11 +54,11 @@ async def test_block_is_refused(): @pytest.mark.asyncio @pytest.mark.parametrize("action", ["modify", "rephrase"]) -async def test_an_intervention_is_not_forwarded_unchanged(action): - """Aporia is saying this should not ship as written, and litellm cannot - apply the modification from this response — so it must not ship it.""" +async def test_an_intervention_is_not_forwarded_unchanged(action: str): + guardrail, _ = _guardrail({"action": action}) + with pytest.raises(HTTPException) as exc: - await _validate(_guardrail(action)) + await _validate(guardrail) assert exc.value.status_code == 400 assert action in str(exc.value.detail) @@ -76,18 +66,11 @@ async def test_an_intervention_is_not_forwarded_unchanged(action): @pytest.mark.asyncio @pytest.mark.parametrize( - "action,include_action", - [ - ("BLOCK", True), - ("blocked", True), - ("", True), - (None, True), - (None, False), - ], + "body", + [{"action": "BLOCK"}, {"action": "blocked"}, {"action": ""}, {"action": None}, {}], ) -async def test_a_verdict_it_cannot_read_is_not_taken_for_permission(action, include_action): - """A response shape change, or an error object with no action at all, used - to forward. A guardrail that cannot tell what it was told is not the one to - decide the content is fine.""" +async def test_a_verdict_it_cannot_read_is_not_taken_for_permission(body: dict[str, str | None]): + guardrail, _ = _guardrail(body) + with pytest.raises(HTTPException): - await _validate(_guardrail(action, include_action=include_action)) + await _validate(guardrail) From 720dbc85481f6d8d56bbddf870a17d0adad4c466 Mon Sep 17 00:00:00 2001 From: L4XB Date: Wed, 23 Sep 2026 20:39:17 +0200 Subject: [PATCH 3/3] test(aporia): name the unreadable-verdict cases The parametrized bodies showed up as body0 to body4 in a failure list --- .../test_litellm/proxy/guardrails/guardrail_hooks/test_aporia.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aporia.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aporia.py index 58272acfa11..86077db7e4e 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aporia.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aporia.py @@ -68,6 +68,7 @@ async def test_an_intervention_is_not_forwarded_unchanged(action: str): @pytest.mark.parametrize( "body", [{"action": "BLOCK"}, {"action": "blocked"}, {"action": ""}, {"action": None}, {}], + ids=["BLOCK", "blocked", "empty", "null", "missing"], ) async def test_a_verdict_it_cannot_read_is_not_taken_for_permission(body: dict[str, str | None]): guardrail, _ = _guardrail(body)