From bcce83a17e599421ec6265891cd6b055906ccb7a Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Tue, 4 Aug 2026 16:46:45 -0700 Subject: [PATCH] fix(guardrails): scan model output on the /openai/v1/responses alias (#35818) The proxy serves POST /openai/v1/responses alongside /responses and /v1/responses, but only the latter two were in API_ROUTE_TO_CALL_TYPES. UnifiedLLMGuardrails.async_post_call_success_hook resolves the call type from request_route, so on the alias it resolved to None and returned the response unscanned; model output reached the client with post-call guardrails never running. The key and team tool allowlist was unenforced on the same alias for the same reason. Register the alias family in API_ROUTE_TO_CALL_TYPES and in LiteLLMRoutes.openai_routes, mirroring how the /openai/v1/realtime aliases are registered, and log a warning at the two points where the unified guardrail skips post-call scanning so a future unmapped route is visible instead of silent. The Responses block of API_ROUTE_TO_CALL_TYPES moves from list to tuple literals because the LIT002 budget rejects net-new mutable-collection construction; the map is read-only, so it is now typed as a Mapping of Sequence and the budgets ratchet down accordingly. --- basedpyright-code-budget.json | 6 +- .../api_route_to_call_types.py | 7 +- litellm/proxy/_types.py | 4 + .../unified_guardrail/unified_guardrail.py | 13 ++ litellm/types/utils.py | 17 +- .../test_unified_guardrail.py | 150 +++++++++++++++++- tests/test_litellm/proxy/test_proxy_server.py | 4 +- type-discipline-budget.json | 4 +- 8 files changed, 188 insertions(+), 17 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 187aeb07f98..614a8e5d2c0 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 29813 + "limit": 29809 }, "reportArgumentType": { "limit": 2645 @@ -60,7 +60,7 @@ "limit": 15849 }, "reportMissingTypeStubs": { - "limit": 41 + "limit": 40 }, "reportOperatorIssue": { "limit": 0 @@ -99,7 +99,7 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 45269 + "limit": 45262 }, "reportUnknownLambdaType": { "limit": 113 diff --git a/litellm/litellm_core_utils/api_route_to_call_types.py b/litellm/litellm_core_utils/api_route_to_call_types.py index 7f1dac544c5..e3562095d7f 100644 --- a/litellm/litellm_core_utils/api_route_to_call_types.py +++ b/litellm/litellm_core_utils/api_route_to_call_types.py @@ -8,6 +8,7 @@ Route patterns may contain placeholders like {agent_id}, {model}, {batch_id}; th match a single path segment when resolving call types for a concrete path. """ +from collections.abc import Sequence from typing import Final from litellm.types.utils import API_ROUTE_TO_CALL_TYPES, CallTypes @@ -30,9 +31,9 @@ def _route_matches_pattern(route: str, pattern: str) -> bool: return True -def get_call_types_for_route(route: str) -> list[CallTypes] | None: +def get_call_types_for_route(route: str) -> Sequence[CallTypes] | None: """ - Get the list of CallTypes for a given API route. + Get the CallTypes for a given API route. Supports both exact keys and dynamic patterns (e.g. /a2a/my-agent/message/send matches /a2a/{agent_id}/message/send). @@ -41,7 +42,7 @@ def get_call_types_for_route(route: str) -> list[CallTypes] | None: route: API route path (e.g., "/chat/completions" or "/a2a/my-pydantic-agent/message/send") Returns: - List of CallTypes for that route, or None if route not found + CallTypes for that route, or None if route not found """ exact: Final = API_ROUTE_TO_CALL_TYPES.get(route, None) if exact is not None: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ef2ac68cd4c..6162f826eef 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -386,12 +386,16 @@ class LiteLLMRoutes(enum.Enum): # responses API "/responses", "/v1/responses", + "/openai/v1/responses", "/responses/{response_id}", "/v1/responses/{response_id}", + "/openai/v1/responses/{response_id}", "/responses/{response_id}/input_items", "/v1/responses/{response_id}/input_items", + "/openai/v1/responses/{response_id}/input_items", "/responses/{response_id}/cancel", "/v1/responses/{response_id}/cancel", + "/openai/v1/responses/{response_id}/cancel", # vector stores "/vector_stores", "/v1/vector_stores", diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index f3c56c7e62b..db86c425c8c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -250,12 +250,25 @@ class UnifiedLLMGuardrails(CustomLogger): call_type = logging_call_type if call_type is None: + verbose_proxy_logger.warning( + "Guardrail '%s' selected for route '%s' but its call type could not be resolved; " + "skipping post-call scanning. Add the route to API_ROUTE_TO_CALL_TYPES.", + guardrail_to_apply.guardrail_name, + user_api_key_dict.request_route, + ) return response if endpoint_guardrail_translation_mappings is None: endpoint_guardrail_translation_mappings = load_guardrail_translation_mappings() if CallTypes(call_type) not in endpoint_guardrail_translation_mappings: + verbose_proxy_logger.warning( + "Guardrail '%s' selected for route '%s' but call type '%s' has no guardrail translation handler; " + "skipping post-call scanning.", + guardrail_to_apply.guardrail_name, + user_api_key_dict.request_route, + call_type, + ) return response endpoint_translation: Final = endpoint_guardrail_translation_mappings[CallTypes(call_type)]() diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 42b7d046be6..5198008687f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -565,7 +565,7 @@ CallTypesLiteral = Literal[ ] # Mapping of API routes to their corresponding call types -API_ROUTE_TO_CALL_TYPES: Final = { +API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { # Chat Completions "/chat/completions": [CallTypes.acompletion, CallTypes.completion], "/v1/chat/completions": [CallTypes.acompletion, CallTypes.completion], @@ -868,12 +868,15 @@ API_ROUTE_TO_CALL_TYPES: Final = { CallTypes.delete_container, ], # Responses API - "/responses": [CallTypes.aresponses, CallTypes.responses], - "/v1/responses": [CallTypes.aresponses, CallTypes.responses], - "/responses/{response_id}": [CallTypes.aresponses, CallTypes.responses], - "/v1/responses/{response_id}": [CallTypes.aresponses, CallTypes.responses], - "/responses/{response_id}/input_items": [CallTypes.alist_input_items], - "/v1/responses/{response_id}/input_items": [CallTypes.alist_input_items], + "/responses": (CallTypes.aresponses, CallTypes.responses), + "/v1/responses": (CallTypes.aresponses, CallTypes.responses), + "/openai/v1/responses": (CallTypes.aresponses, CallTypes.responses), + "/responses/{response_id}": (CallTypes.aresponses, CallTypes.responses), + "/v1/responses/{response_id}": (CallTypes.aresponses, CallTypes.responses), + "/openai/v1/responses/{response_id}": (CallTypes.aresponses, CallTypes.responses), + "/responses/{response_id}/input_items": (CallTypes.alist_input_items,), + "/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), + "/openai/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), # Realtime API "/realtime": [CallTypes.arealtime], "/v1/realtime": [CallTypes.arealtime], diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index bf904dbe394..8a551f749d0 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -1,5 +1,7 @@ """Tests for unified guardrail.""" +import logging + import pytest import litellm @@ -8,6 +10,8 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route +from litellm.llms import load_guardrail_translation_mappings from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, @@ -18,12 +22,15 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( from litellm.llms.openai.chat.guardrail_translation.handler import ( OpenAIChatCompletionsHandler, ) +from litellm.llms.openai.responses.guardrail_translation.handler import ( + OpenAIResponsesHandler, +) from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse from litellm.llms.mistral.ocr.guardrail_translation.handler import OCRHandler from litellm.proxy._experimental.mcp_server.guardrail_translation.handler import ( MCPGuardrailTranslationHandler, ) -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail import ( unified_guardrail as unified_module, ) @@ -31,6 +38,7 @@ from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrai UnifiedLLMGuardrails, ) from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import CallTypes, Delta, ModelResponseStream, StreamingChoices @@ -75,6 +83,8 @@ def _inject_mcp_handler_mapping(): CallTypes.anthropic_messages: _NoopTranslation, CallTypes.ocr: OCRHandler, CallTypes.aocr: OCRHandler, + CallTypes.responses: OpenAIResponsesHandler, + CallTypes.aresponses: OpenAIResponsesHandler, } yield unified_module.endpoint_guardrail_translation_mappings = None @@ -486,6 +496,144 @@ class TestUnifiedLLMGuardrails: f"Expected non-empty content for every streamed chunk." ) + class TestResponsesRouteAliases: + """Every /responses path alias that serves model output must scan it. + + ``async_post_call_success_hook`` resolves the call type from + ``request_route`` via ``API_ROUTE_TO_CALL_TYPES``. A route missing from + that map resolves to ``None`` and the hook returns the response + unscanned, so an alias that the proxy serves but the map omits is a + silent post-call guardrail bypass. + """ + + @staticmethod + def _responses_api_response() -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id="resp_lit4979", + created_at=1234567890, + model="gpt-4o", + object="response", + status="completed", + output=[ + { + "type": "message", + "id": "msg_lit4979", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Paris"}], + } + ], + ) + + @pytest.mark.parametrize( + "request_route", + [ + "/responses", + "/v1/responses", + "/openai/v1/responses", + "/responses/{response_id}", + "/v1/responses/{response_id}", + "/openai/v1/responses/{response_id}", + ], + ) + @pytest.mark.asyncio + async def test_post_call_scans_output_on_every_registered_alias( + self, request_route: str + ) -> None: + handler = UnifiedLLMGuardrails() + guardrail = RecordingGuardrail() + + await handler.async_post_call_success_hook( + data={"guardrail_to_apply": guardrail, "model": "gpt-4o"}, + user_api_key_dict=UserAPIKeyAuth( + api_key="test-key", request_route=request_route + ), + response=self._responses_api_response(), + ) + + assert guardrail.apply_calls, ( + f"guardrail never ran for request_route={request_route!r}; model " + f"output reached the client unscanned" + ) + assert guardrail.apply_calls[0]["input_type"] == "response" + assert guardrail.apply_calls[0]["inputs"]["texts"] == ["Paris"] + + @pytest.mark.parametrize( + "route, expected", + [ + ("/openai/v1/responses", (CallTypes.aresponses, CallTypes.responses)), + ( + "/openai/v1/responses/resp_abc", + (CallTypes.aresponses, CallTypes.responses), + ), + ( + "/openai/v1/responses/resp_abc/input_items", + (CallTypes.alist_input_items,), + ), + ], + ) + def test_openai_prefixed_aliases_resolve_like_canonical_routes( + self, route: str, expected: tuple[CallTypes, ...] + ) -> None: + assert tuple(get_call_types_for_route(route) or ()) == expected + + def test_responses_handler_is_registered_in_the_real_registry(self) -> None: + mappings = load_guardrail_translation_mappings() + assert CallTypes.aresponses in mappings + assert CallTypes.responses in mappings + + @pytest.mark.asyncio + async def test_unresolvable_route_skips_scanning_and_says_so( + self, caplog: pytest.LogCaptureFixture + ) -> None: + handler = UnifiedLLMGuardrails() + guardrail = RecordingGuardrail() + + with caplog.at_level(logging.WARNING): + result = await handler.async_post_call_success_hook( + data={"guardrail_to_apply": guardrail, "model": "gpt-4o"}, + user_api_key_dict=UserAPIKeyAuth( + api_key="test-key", request_route="/cursor/chat/completions" + ), + response=self._responses_api_response(), + ) + + assert not guardrail.apply_calls + assert result is not None + assert "call type could not be resolved" in caplog.text + assert "/cursor/chat/completions" in caplog.text + + @pytest.mark.asyncio + async def test_call_type_without_handler_skips_scanning_and_says_so( + self, caplog: pytest.LogCaptureFixture + ) -> None: + handler = UnifiedLLMGuardrails() + guardrail = RecordingGuardrail() + + with caplog.at_level(logging.WARNING): + await handler.async_post_call_success_hook( + data={"guardrail_to_apply": guardrail, "model": "gpt-4o"}, + user_api_key_dict=UserAPIKeyAuth( + api_key="test-key", request_route="/v1/chat/completions" + ), + response=self._responses_api_response(), + ) + + assert not guardrail.apply_calls + assert "has no guardrail translation handler" in caplog.text + + def test_openai_prefixed_aliases_are_authorized_like_canonical_routes(self) -> None: + openai_routes = LiteLLMRoutes.openai_routes.value + for route in ( + "/openai/v1/responses", + "/openai/v1/responses/{response_id}", + "/openai/v1/responses/{response_id}/input_items", + ): + assert route in openai_routes, ( + f"{route!r} missing from LiteLLMRoutes.openai_routes; team and " + f"key-scoped users get 403 on this alias" + ) + class TestOCRGuardrailE2E: """End-to-end tests: UnifiedLLMGuardrails -> OCRHandler.""" diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index c7aa376a2f7..ede93dc0c58 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -9221,7 +9221,9 @@ def test_realtime_websocket_route_aliases_registered(): f"{expected!r} missing from LiteLLMRoutes.openai_routes; " f"non-admin / team / key-scoped users will get 403 on this path." ) - assert API_ROUTE_TO_CALL_TYPES.get(expected) == [CallTypes.arealtime], ( + assert tuple(API_ROUTE_TO_CALL_TYPES.get(expected) or ()) == ( + CallTypes.arealtime, + ), ( f"{expected!r} missing from API_ROUTE_TO_CALL_TYPES; call-type " f"resolution will return None and break call-type-aware features." ) diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 254f085831c..582c0d662e6 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 23350 + "limit": 23348 }, "LIT002": { - "limit": 27234 + "limit": 27227 }, "LIT003": { "limit": 292