From 449b37364bec6271069330d35ffe6e128bb77f2a Mon Sep 17 00:00:00 2001 From: Chau Vu / CPF-FAMILY <58875435+chauvuusvn@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:26:13 +0700 Subject: [PATCH 1/2] fix(proxy): add /openai/responses router routes for Azure SDK Responses API (Fixes #45033) - Register /openai/responses, /compact, /input_tokens, /{id}, /cancel, and /{id}/input_items endpoints on the proxy router - Update RESPONSES_API_PATHS, _MANAGED_MODEL_ROUTES, _UNBILLED_ROUTES, and LiteLLMRoutes - Prevent AzureOpenAI Responses API calls from falling into wildcard pass-through - Add unit test coverage in tests/proxy_unit_tests/test_openai_responses_routes.py --- litellm/proxy/_types.py | 5 +++ .../auth/managed_authorization.py | 1 + .../proxy/common_utils/custom_openapi_spec.py | 2 +- .../proxy/response_api_endpoints/endpoints.py | 39 +++++++++++++++++++ .../spend_tracking/budget_reservation.py | 1 + .../test_openai_responses_routes.py | 27 +++++++++++++ 6 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 tests/proxy_unit_tests/test_openai_responses_routes.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f009390576a..ba8d9061d44 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -454,18 +454,23 @@ class LiteLLMRoutes(enum.Enum): "/responses", "/v1/responses", "/openai/v1/responses", + "/openai/responses", "/responses/{response_id}", "/v1/responses/{response_id}", "/openai/v1/responses/{response_id}", + "/openai/responses/{response_id}", "/responses/{response_id}/input_items", "/v1/responses/{response_id}/input_items", "/openai/v1/responses/{response_id}/input_items", + "/openai/responses/{response_id}/input_items", "/responses/{response_id}/cancel", "/v1/responses/{response_id}/cancel", "/openai/v1/responses/{response_id}/cancel", + "/openai/responses/{response_id}/cancel", "/responses/input_tokens", "/v1/responses/input_tokens", "/openai/v1/responses/input_tokens", + "/openai/responses/input_tokens", # vector stores "/vector_stores", "/v1/vector_stores", diff --git a/litellm/proxy/agent_endpoints/auth/managed_authorization.py b/litellm/proxy/agent_endpoints/auth/managed_authorization.py index 64e50cd74e7..0ea4060bcee 100644 --- a/litellm/proxy/agent_endpoints/auth/managed_authorization.py +++ b/litellm/proxy/agent_endpoints/auth/managed_authorization.py @@ -37,6 +37,7 @@ _MANAGED_MODEL_ROUTES: Final = frozenset( ) | frozenset( ( "/openai/v1/responses", + "/openai/responses", "/v2/rerank", "/claude_code_gateway/v1/messages", "/claude_code_gateway/v1/messages/count_tokens", diff --git a/litellm/proxy/common_utils/custom_openapi_spec.py b/litellm/proxy/common_utils/custom_openapi_spec.py index c1d4442893a..f79dce9cff2 100644 --- a/litellm/proxy/common_utils/custom_openapi_spec.py +++ b/litellm/proxy/common_utils/custom_openapi_spec.py @@ -31,7 +31,7 @@ class CustomOpenAPISpec: "/openai/deployments/{model}/embeddings", ] - RESPONSES_API_PATHS = ["/v1/responses", "/responses", "/openai/v1/responses"] + RESPONSES_API_PATHS = ["/v1/responses", "/responses", "/openai/v1/responses", "/openai/responses"] @staticmethod def _as_object(node: JsonValue) -> JsonObject: diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 7f9fcca369b..f6f72971b47 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -213,6 +213,12 @@ async def _resolve_cursor_model_variant_before_auth(request: Request) -> None: tags=["responses"], responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS, ) +@router.post( + "/openai/responses", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], + responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS, +) async def responses_api( request: Request, fastapi_response: Response, @@ -696,6 +702,12 @@ async def cursor_chat_completions( tags=["responses"], responses=RESPONSES_API_RESPONSE_SCHEMAS, ) +@router.get( + "/openai/responses/{response_id}", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], + responses=RESPONSES_API_RESPONSE_SCHEMAS, +) async def get_response( response_id: str, request: Request, @@ -812,6 +824,12 @@ async def get_response( tags=["responses"], responses=DELETE_RESPONSE_SCHEMAS, ) +@router.delete( + "/openai/responses/{response_id}", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], + responses=DELETE_RESPONSE_SCHEMAS, +) async def delete_response( response_id: str, request: Request, @@ -921,6 +939,12 @@ async def delete_response( tags=["responses"], responses=RESPONSE_ITEM_LIST_SCHEMAS, ) +@router.get( + "/openai/responses/{response_id}/input_items", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], + responses=RESPONSE_ITEM_LIST_SCHEMAS, +) async def get_response_input_items( response_id: str, request: Request, @@ -989,6 +1013,11 @@ async def get_response_input_items( dependencies=[Depends(user_api_key_auth)], tags=["responses"], ) +@router.post( + "/openai/responses/compact", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], +) async def compact_response( request: Request, fastapi_response: Response, @@ -1144,6 +1173,11 @@ def _responses_input_as_token_count_messages( dependencies=(_user_api_key_auth_dep,), tags=_RESPONSES_TAGS, ) +@router.post( + "/openai/responses/input_tokens", + dependencies=(_user_api_key_auth_dep,), + tags=_RESPONSES_TAGS, +) async def responses_input_tokens( request: Request, token_counter: _TokenCounter = _token_counter_dep, @@ -1217,6 +1251,11 @@ async def responses_input_tokens( dependencies=[Depends(user_api_key_auth)], tags=["responses"], ) +@router.post( + "/openai/responses/{response_id}/cancel", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], +) async def cancel_response( response_id: str, request: Request, diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 4c9df95cd8d..96f78c164cd 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -236,6 +236,7 @@ _UNBILLED_ROUTES: Final[frozenset[str]] = frozenset( "/responses/input_tokens", "/v1/responses/input_tokens", "/openai/v1/responses/input_tokens", + "/openai/responses/input_tokens", } ) _TOKEN_COUNTING_SEGMENTS: Final[frozenset[str]] = frozenset({"count_tokens", "count-tokens"}) diff --git a/tests/proxy_unit_tests/test_openai_responses_routes.py b/tests/proxy_unit_tests/test_openai_responses_routes.py new file mode 100644 index 00000000000..dc83a1fb257 --- /dev/null +++ b/tests/proxy_unit_tests/test_openai_responses_routes.py @@ -0,0 +1,27 @@ +from litellm.proxy._types import LiteLLMRoutes +from litellm.proxy.agent_endpoints.auth.managed_authorization import _MANAGED_MODEL_ROUTES +from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec +from litellm.proxy.response_api_endpoints.endpoints import router +from litellm.proxy.spend_tracking.budget_reservation import _UNBILLED_ROUTES + + +def test_openai_responses_routes_registered(): + """Verify that Azure-style /openai/responses endpoints are registered on the router.""" + routes = {getattr(r, "path", None) for r in router.routes} + + expected_routes = [ + "/openai/responses", + "/openai/responses/{response_id}", + "/openai/responses/{response_id}/input_items", + "/openai/responses/compact", + "/openai/responses/input_tokens", + "/openai/responses/{response_id}/cancel", + ] + + for expected in expected_routes: + assert expected in routes, f"Route {expected} missing from router.routes!" + + assert "/openai/responses" in CustomOpenAPISpec.RESPONSES_API_PATHS + assert "/openai/responses" in _MANAGED_MODEL_ROUTES + assert "/openai/responses/input_tokens" in _UNBILLED_ROUTES + assert "/openai/responses" in LiteLLMRoutes.openai_routes.value From 4064e59aa1c18099c53866880ff8ff5f4e9a1d0c Mon Sep 17 00:00:00 2001 From: Chau Vu / CPF-FAMILY <58875435+chauvuusvn@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:47:06 +0700 Subject: [PATCH 2/2] fix(proxy): add /openai/responses to API_ROUTE_TO_CALL_TYPES and Prometheus route normalization - Map /openai/responses routes in API_ROUTE_TO_CALL_TYPES for tool allowlist and guardrail validation - Add regex normalization patterns for /openai/responses/{response_id} in normalize_request_route - Expand test suite in tests/proxy_unit_tests/test_openai_responses_routes.py --- litellm/proxy/auth/auth_utils.py | 3 +++ litellm/types/utils.py | 3 +++ .../test_openai_responses_routes.py | 16 ++++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 2bd10b90101..9e603af01dd 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -857,6 +857,9 @@ def normalize_request_route(route: str) -> str: (r"^(/(?:openai/)?v1/responses)/([^/]+)(/input_items)$", r"\1/{response_id}\3"), (r"^(/(?:openai/)?v1/responses)/([^/]+)(/cancel)$", r"\1/{response_id}\3"), (r"^(/(?:openai/)?v1/responses)/([^/]+)$", r"\1/{response_id}"), + (r"^(/openai/responses)/([^/]+)(/input_items)$", r"\1/{response_id}\3"), + (r"^(/openai/responses)/([^/]+)(/cancel)$", r"\1/{response_id}\3"), + (r"^(/openai/responses)/([^/]+)$", r"\1/{response_id}"), (r"^(/responses)/([^/]+)(/input_items)$", r"\1/{response_id}\3"), (r"^(/responses)/([^/]+)(/cancel)$", r"\1/{response_id}\3"), (r"^(/responses)/([^/]+)$", r"\1/{response_id}"), diff --git a/litellm/types/utils.py b/litellm/types/utils.py index a99c22e9479..7ebbe04b197 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1038,12 +1038,15 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { "/responses": (CallTypes.aresponses, CallTypes.responses), "/v1/responses": (CallTypes.aresponses, CallTypes.responses), "/openai/v1/responses": (CallTypes.aresponses, CallTypes.responses), + "/openai/responses": (CallTypes.aresponses, CallTypes.responses), "/responses/{response_id}": (CallTypes.aresponses, CallTypes.responses), "/v1/responses/{response_id}": (CallTypes.aresponses, CallTypes.responses), "/openai/v1/responses/{response_id}": (CallTypes.aresponses, CallTypes.responses), + "/openai/responses/{response_id}": (CallTypes.aresponses, CallTypes.responses), "/responses/{response_id}/input_items": (CallTypes.alist_input_items,), "/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), "/openai/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), + "/openai/responses/{response_id}/input_items": (CallTypes.alist_input_items,), # Realtime API "/realtime": [CallTypes.arealtime], "/v1/realtime": [CallTypes.arealtime], diff --git a/tests/proxy_unit_tests/test_openai_responses_routes.py b/tests/proxy_unit_tests/test_openai_responses_routes.py index dc83a1fb257..21c7865a7fb 100644 --- a/tests/proxy_unit_tests/test_openai_responses_routes.py +++ b/tests/proxy_unit_tests/test_openai_responses_routes.py @@ -1,8 +1,10 @@ from litellm.proxy._types import LiteLLMRoutes from litellm.proxy.agent_endpoints.auth.managed_authorization import _MANAGED_MODEL_ROUTES +from litellm.proxy.auth.auth_utils import normalize_request_route from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec from litellm.proxy.response_api_endpoints.endpoints import router from litellm.proxy.spend_tracking.budget_reservation import _UNBILLED_ROUTES +from litellm.types.utils import API_ROUTE_TO_CALL_TYPES, CallTypes def test_openai_responses_routes_registered(): @@ -25,3 +27,17 @@ def test_openai_responses_routes_registered(): assert "/openai/responses" in _MANAGED_MODEL_ROUTES assert "/openai/responses/input_tokens" in _UNBILLED_ROUTES assert "/openai/responses" in LiteLLMRoutes.openai_routes.value + + +def test_openai_responses_api_route_to_call_types(): + """Verify that /openai/responses mappings exist in API_ROUTE_TO_CALL_TYPES for guardrails/tools.""" + assert API_ROUTE_TO_CALL_TYPES.get("/openai/responses") == (CallTypes.aresponses, CallTypes.responses) + assert API_ROUTE_TO_CALL_TYPES.get("/openai/responses/{response_id}") == (CallTypes.aresponses, CallTypes.responses) + assert API_ROUTE_TO_CALL_TYPES.get("/openai/responses/{response_id}/input_items") == (CallTypes.alist_input_items,) + + +def test_openai_responses_metrics_normalization(): + """Verify that dynamic response IDs are normalized to prevent Prometheus high-cardinality metrics.""" + assert normalize_request_route("/openai/responses/resp-123456") == "/openai/responses/{response_id}" + assert normalize_request_route("/openai/responses/resp-123456/cancel") == "/openai/responses/{response_id}/cancel" + assert normalize_request_route("/openai/responses/resp-123456/input_items") == "/openai/responses/{response_id}/input_items"