diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e53b5d4ad52..326548dd72d 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -352,18 +352,23 @@ class LiteLLMRoutes(enum.Enum): "/responses", "/v1/responses", "/openai/v1/responses", + "/openai/responses", "/responses/{response_id}", "/v1/responses/{response_id}", "/openai/v1/responses/{response_id}", + "/openai/responses/{response_id}", "/responses/{response_id}/input_items", "/v1/responses/{response_id}/input_items", "/openai/v1/responses/{response_id}/input_items", + "/openai/responses/{response_id}/input_items", "/responses/{response_id}/cancel", "/v1/responses/{response_id}/cancel", "/openai/v1/responses/{response_id}/cancel", + "/openai/responses/{response_id}/cancel", "/responses/input_tokens", "/v1/responses/input_tokens", "/openai/v1/responses/input_tokens", + "/openai/responses/input_tokens", # vector stores "/vector_stores", "/v1/vector_stores", diff --git a/litellm/proxy/agent_endpoints/auth/managed_authorization.py b/litellm/proxy/agent_endpoints/auth/managed_authorization.py index 57e99edecba..0b3e11b19ea 100644 --- a/litellm/proxy/agent_endpoints/auth/managed_authorization.py +++ b/litellm/proxy/agent_endpoints/auth/managed_authorization.py @@ -47,6 +47,7 @@ _MANAGED_MODEL_ROUTES: Final = frozenset( ) | frozenset( ( "/openai/v1/responses", + "/openai/responses", "/v2/rerank", "/claude_code_gateway/v1/messages", "/claude_code_gateway/v1/messages/count_tokens", diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 2b5fc6d877e..f29dd4e9e66 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -1037,12 +1037,9 @@ def normalize_request_route(route: str) -> str: # Format: (regex_pattern, replacement_template) patterns: Final = [ # Responses API - must come before generic patterns - (r"^(/(?:openai/)?v1/responses)/([^/]+)(/input_items)$", r"\1/{response_id}\3"), - (r"^(/(?:openai/)?v1/responses)/([^/]+)(/cancel)$", r"\1/{response_id}\3"), - (r"^(/(?:openai/)?v1/responses)/([^/]+)$", r"\1/{response_id}"), - (r"^(/responses)/([^/]+)(/input_items)$", r"\1/{response_id}\3"), - (r"^(/responses)/([^/]+)(/cancel)$", r"\1/{response_id}\3"), - (r"^(/responses)/([^/]+)$", r"\1/{response_id}"), + (r"^(/(?:openai/)?(?:v1/)?responses)/([^/]+)(/input_items)$", r"\1/{response_id}\3"), + (r"^(/(?:openai/)?(?:v1/)?responses)/([^/]+)(/cancel)$", r"\1/{response_id}\3"), + (r"^(/(?:openai/)?(?:v1/)?responses)/([^/]+)$", r"\1/{response_id}"), # Threads API ( r"^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/steps)/([^/]+)$", diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 125bfab4edf..444060ba1eb 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -103,6 +103,7 @@ _UNBILLED_ROUTES: Final[frozenset[str]] = frozenset( "/responses/input_tokens", "/v1/responses/input_tokens", "/openai/v1/responses/input_tokens", + "/openai/responses/input_tokens", } ) _TOKEN_COUNTING_SEGMENTS: Final[frozenset[str]] = frozenset({"count_tokens", "count-tokens"}) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index c4cd7c2d411..0751416f800 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -312,6 +312,12 @@ async def _resolve_cursor_model_variant_before_auth(request: Request) -> None: tags=["responses"], responses=RESPONSES_API_RESPONSE_SCHEMAS, ) +@router.post( + "/openai/responses", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], + include_in_schema=False, +) async def responses_api( request: Request, fastapi_response: Response, @@ -809,6 +815,12 @@ async def cursor_chat_completions( responses=RESPONSES_API_RESPONSE_SCHEMAS, openapi_extra=_RETRIEVE_QUERY_OPENAPI, ) +@router.get( + "/openai/responses/{response_id}", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], + include_in_schema=False, +) async def get_response( response_id: str, request: Request, @@ -1240,6 +1252,12 @@ async def _bill_finished_stream( tags=["responses"], responses=DELETE_RESPONSE_SCHEMAS, ) +@router.delete( + "/openai/responses/{response_id}", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], + include_in_schema=False, +) async def delete_response( response_id: str, request: Request, @@ -1356,6 +1374,12 @@ async def delete_response( tags=["responses"], responses=RESPONSE_ITEM_LIST_SCHEMAS, ) +@router.get( + "/openai/responses/{response_id}/input_items", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], + include_in_schema=False, +) async def get_response_input_items( response_id: str, request: Request, @@ -1424,6 +1448,12 @@ async def get_response_input_items( dependencies=[Depends(user_api_key_auth)], tags=["responses"], ) +@router.post( + "/openai/responses/compact", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], + include_in_schema=False, +) async def compact_response( request: Request, fastapi_response: Response, @@ -1736,6 +1766,12 @@ def _responses_input_as_token_count_messages( dependencies=(_user_api_key_auth_dep,), tags=_RESPONSES_TAGS, ) +@router.post( + "/openai/responses/input_tokens", + dependencies=(_user_api_key_auth_dep,), + tags=_RESPONSES_TAGS, + include_in_schema=False, +) async def responses_input_tokens( request: Request, token_counter: _TokenCounter = _token_counter_dep, @@ -1868,6 +1904,12 @@ async def responses_input_tokens( dependencies=[Depends(user_api_key_auth)], tags=["responses"], ) +@router.post( + "/openai/responses/{response_id}/cancel", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], + include_in_schema=False, +) async def cancel_response( response_id: str, request: Request, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5a14dc95e7c..10de5c31661 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1140,12 +1140,15 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { "/responses": (CallTypes.aresponses, CallTypes.responses), "/v1/responses": (CallTypes.aresponses, CallTypes.responses), "/openai/v1/responses": (CallTypes.aresponses, CallTypes.responses), + "/openai/responses": (CallTypes.aresponses, CallTypes.responses), "/responses/{response_id}": (CallTypes.aresponses, CallTypes.responses), "/v1/responses/{response_id}": (CallTypes.aresponses, CallTypes.responses), "/openai/v1/responses/{response_id}": (CallTypes.aresponses, CallTypes.responses), + "/openai/responses/{response_id}": (CallTypes.aresponses, CallTypes.responses), "/responses/{response_id}/input_items": (CallTypes.alist_input_items,), "/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), "/openai/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), + "/openai/responses/{response_id}/input_items": (CallTypes.alist_input_items,), # Realtime API "/realtime": (CallTypes.arealtime,), "/v1/realtime": (CallTypes.arealtime,), diff --git a/tests/unit/integrations/test_prometheus_labels.py b/tests/unit/integrations/test_prometheus_labels.py index 04eb7c3a5a4..339c021aa30 100644 --- a/tests/unit/integrations/test_prometheus_labels.py +++ b/tests/unit/integrations/test_prometheus_labels.py @@ -463,6 +463,9 @@ def test_route_normalization_for_sub_routes(): "/openai/v1/responses/id4/input_items", "/openai/v1/responses/{response_id}/input_items", ), + ("/openai/responses/id5/cancel", "/openai/responses/{response_id}/cancel"), + ("/openai/responses/id6/input_items", "/openai/responses/{response_id}/input_items"), + ("/openai/responses/id7", "/openai/responses/{response_id}"), ] for original, expected in sub_routes: diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index b7d0d9e62cd..d036acec4e0 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -5273,7 +5273,12 @@ class TestResponsesInputTokens: assert token_request.messages == [{"role": "user", "content": "Hello, how are you?"}] def test_every_route_alias_is_registered(self): - for path in ("/v1/responses/input_tokens", "/responses/input_tokens", "/openai/v1/responses/input_tokens"): + for path in ( + "/v1/responses/input_tokens", + "/responses/input_tokens", + "/openai/v1/responses/input_tokens", + "/openai/responses/input_tokens", + ): response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, path=path) assert response.status_code == 200, f"{path}: {response.status_code} {response.text}" @@ -5498,7 +5503,13 @@ class TestResponsesInputTokens: assert sent == [{"model": "gpt-4o-mini", "input": "hi"}] @pytest.mark.parametrize( - "path", ["/v1/responses/input_tokens", "/responses/input_tokens", "/openai/v1/responses/input_tokens"] + "path", + [ + "/v1/responses/input_tokens", + "/responses/input_tokens", + "/openai/v1/responses/input_tokens", + "/openai/responses/input_tokens", + ], ) def test_count_logs_one_line_with_the_callers_call_id_model_and_count( self, @@ -5977,3 +5988,125 @@ def test_input_token_count_reads_a_compaction_item_as_its_summary(): assert _responses_input_as_token_count_messages(_COMPACTED_INPUT, None) == ( _responses_input_as_token_count_messages(_SUMMARY_INPUT, None) ) + + +_AZURE_SDK_RESOURCE: Final = "my-resource.openai.azure.com" +_AZURE_SDK_RESPONSE: Final = { + "id": "resp_azure_sdk", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "my-deployment", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Done.", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 10, "output_tokens": 2, "total_tokens": 12}, +} +_AZURE_SDK_RESPONSE_PATHS: Final = ( + ("GET", "/openai/responses/{id}", ""), + ("DELETE", "/openai/responses/{id}", ""), + ("GET", "/openai/responses/{id}/input_items", "/input_items"), + ("POST", "/openai/responses/{id}/cancel", "/cancel"), +) + + +def _azure_lifecycle_response(request: httpx.Request) -> httpx.Response: + if request.method == "DELETE": + return httpx.Response(200, json={"id": "resp_azure_sdk", "object": "response", "deleted": True}) + if request.url.path.endswith("/input_items"): + return httpx.Response(200, json={"object": "list", "data": [], "has_more": False}) + return httpx.Response(200, json=_AZURE_SDK_RESPONSE) + + +@pytest.fixture +def azure_sdk_proxy(monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter) -> TestClient: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm.proxy.proxy_server, "master_key", MASTER_KEY) + monkeypatch.setattr(litellm.proxy.proxy_server, "prisma_client", None) + monkeypatch.setattr(litellm.proxy.proxy_server, "general_settings", {}) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "my-model", + "litellm_params": { + "model": "azure/my-deployment", + "api_base": f"https://{_AZURE_SDK_RESOURCE}", + "api_key": "azure-deployment-key", + "api_version": "2025-04-01-preview", + }, + } + ] + ), + ) + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.route(host="testserver").pass_through() + return TestClient(app, headers={"Authorization": f"Bearer {MASTER_KEY}"}) + + +def test_azure_sdk_responses_path_reaches_the_azure_deployment( + azure_sdk_proxy: TestClient, respx_mock: respx.MockRouter +) -> None: + azure: Final = respx_mock.post(host=_AZURE_SDK_RESOURCE, path__regex=r"/responses$").respond( + json=_AZURE_SDK_RESPONSE + ) + + response: Final = azure_sdk_proxy.post( + "/openai/responses?api-version=2025-04-01-preview", json={"model": "my-model", "input": "hi"} + ) + + assert response.status_code == 200, response.text + assert response.json()["output"][0]["content"][0]["text"] == "Done." + assert azure.call_count == 1 + assert azure.calls[0].request.headers["api-key"] == "azure-deployment-key" + assert json.loads(azure.calls[0].request.content)["model"] == "my-deployment" + + +@pytest.mark.parametrize(("method", "path", "azure_suffix"), _AZURE_SDK_RESPONSE_PATHS) +def test_azure_sdk_response_lifecycle_paths_reach_the_creating_deployment( + azure_sdk_proxy: TestClient, respx_mock: respx.MockRouter, method: str, path: str, azure_suffix: str +) -> None: + respx_mock.post(host=_AZURE_SDK_RESOURCE, path__regex=r"/responses$").respond(json=_AZURE_SDK_RESPONSE) + lifecycle: Final = respx_mock.route( + host=_AZURE_SDK_RESOURCE, path__regex=rf"/responses/resp_azure_sdk{azure_suffix}$" + ).mock(side_effect=_azure_lifecycle_response) + created: Final = azure_sdk_proxy.post("/openai/responses", json={"model": "my-model", "input": "hi"}) + assert created.status_code == 200, created.text + + response: Final = azure_sdk_proxy.request(method, path.format(id=created.json()["id"])) + + assert response.status_code == 200, response.text + assert [call.request.method for call in lifecycle.calls] == [method] + + +@pytest.mark.parametrize( + ("method", "path"), + ( + ("POST", "/openai/responses"), + ("POST", "/openai/responses/compact"), + ("POST", "/openai/responses/input_tokens"), + *((method, path.format(id="resp_azure_sdk")) for method, path, _ in _AZURE_SDK_RESPONSE_PATHS), + ), +) +def test_azure_sdk_responses_paths_require_a_virtual_key( + azure_sdk_proxy: TestClient, respx_mock: respx.MockRouter, method: str, path: str +) -> None: + azure: Final = respx_mock.route(host=_AZURE_SDK_RESOURCE).respond(json=_AZURE_SDK_RESPONSE) + + response: Final = azure_sdk_proxy.request( + method, path, json={"model": "my-model", "input": "hi"}, headers={"Authorization": ""} + ) + + assert response.status_code == 401, response.text + assert not azure.called