From 93266c971e33834954eee19217028a7cebaae8ed Mon Sep 17 00:00:00 2001 From: Airia Open Source Date: Tue, 3 Nov 2026 02:57:25 -0600 Subject: [PATCH] test(guardrails): pin that only MCP hooks mark an Airia evaluation as a tool call Cover the cases the marker must not appear in: during_call and post_call evaluations, and client request data shaped like an MCP call (a top-level mcp_tool_name, mcp_tool_call_metadata, or metadata.mcp_tool_name). Also pin the fallback name used when an MCP evaluation carries none, and that a hook that does not run leaves no marker for the next evaluation. --- .../guardrails/guardrail_hooks/test_airia.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_airia.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_airia.py index 4eb9e2fcaf0..76e330cca73 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_airia.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_airia.py @@ -229,6 +229,57 @@ async def test_an_mcp_evaluation_does_not_mark_the_next_model_evaluation_as_a_to assert _post_mock(guardrail).call_args.kwargs["json"]["mcp_tool_name"] is None +@pytest.mark.asyncio +@pytest.mark.parametrize( + "marker", + [ + {"mcp_tool_name": "send_email"}, + {"mcp_tool_call_metadata": {"name": "send_email", "arguments": {}}}, + {"metadata": {"mcp_tool_name": "send_email"}}, + ], +) +async def test_client_supplied_mcp_markers_do_not_mark_model_traffic_as_a_tool_call( + marker: dict[str, object], +) -> None: + guardrail: Final = _make_guardrail_with_response({"action": "NONE"}, event_hook="pre_call", default_on=True) + + await UnifiedLLMGuardrails().async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # annotates data as a bare dict + user_api_key_dict=UserAPIKeyAuth(api_key="test"), + cache=DualCache(), + data={"guardrail_to_apply": guardrail, "messages": [{"role": "user", "content": "hello"}], **marker}, + call_type="completion", + ) + + assert _post_mock(guardrail).call_args.kwargs["json"]["mcp_tool_name"] is None + + +@pytest.mark.asyncio +async def test_a_hook_that_does_not_run_leaves_no_marker_for_the_next_evaluation() -> None: + guardrail: Final = _make_guardrail_with_response({"action": "NONE"}, event_hook="pre_call", default_on=True) + tool_call: Final[dict[str, object]] = {"mcp_tool_name": "send_email"} + + assert not guardrail.should_run_guardrail(data=tool_call, event_type=GuardrailEventHooks.pre_mcp_call) + await guardrail.apply_guardrail(inputs=_inputs(), request_data=tool_call, input_type="request") + + assert _post_mock(guardrail).call_args.kwargs["json"]["mcp_tool_name"] is None + + +@pytest.mark.asyncio +async def test_an_mcp_evaluation_without_a_tool_name_still_names_one_so_tool_filters_apply( + monkeypatch: pytest.MonkeyPatch, +) -> None: + guardrail: Final = _make_guardrail_with_response({"action": "NONE"}, event_hook="post_mcp_call", default_on=True) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + + await ProxyLogging(user_api_key_cache=UserApiKeyCache()).post_mcp_call_hook( + response=CallToolResult(content=[TextContent(type="text", text="ok")]), + request_data={"mcp_tool_call_metadata": {"arguments": {}}}, + user_api_key_dict=UserAPIKeyAuth(api_key="test"), + ) + + assert _post_mock(guardrail).call_args.kwargs["json"]["mcp_tool_name"] == "unknown" + + def _during_call_request() -> dict[str, object]: return {"model": "gpt-4o", "messages": [{"role": "user", "content": "mail ada@example.com"}]} @@ -254,6 +305,7 @@ async def test_during_call_enforces_every_intervention_as_a_block( assert excinfo.value.blocked_content is True assert _post_mock(guardrail).call_args.kwargs["json"]["input_type"] == "request" + assert _post_mock(guardrail).call_args.kwargs["json"]["mcp_tool_name"] is None @pytest.mark.asyncio @@ -268,6 +320,7 @@ async def test_during_call_allow_lets_the_request_through(monkeypatch: pytest.Mo assert request["messages"] == _during_call_request()["messages"] _post_mock(guardrail).assert_called_once() + assert _post_mock(guardrail).call_args.kwargs["json"]["mcp_tool_name"] is None @pytest.mark.asyncio @@ -571,6 +624,7 @@ async def test_streaming_redaction_actually_reaches_the_client() -> None: request_data={ # mutable-ok: one-shot request_data the framework requires as a plain dict "messages": [{"role": "user", "content": "share your contact"}], "stream": True, + "mcp_tool_name": "send_email", }, guardrail_to_apply=guardrail, ): @@ -581,6 +635,7 @@ async def test_streaming_redaction_actually_reaches_the_client() -> None: assert "[EmailAddress1]" in assembled assert "ada@example.com" not in assembled _post_mock(guardrail).assert_called_once() + assert _post_mock(guardrail).call_args.kwargs["json"]["mcp_tool_name"] is None def _streamed_request(route: str | None) -> dict[str, object]: