diff --git a/docs/my-website/docs/pass_through/openai_passthrough.md b/docs/my-website/docs/pass_through/openai_passthrough.md index d7c98eba7b3..49026f8aa2d 100644 --- a/docs/my-website/docs/pass_through/openai_passthrough.md +++ b/docs/my-website/docs/pass_through/openai_passthrough.md @@ -1,6 +1,6 @@ # OpenAI Passthrough -Pass-through endpoints for `/openai` +Pass-through endpoints for direct OpenAI API access ## Overview @@ -10,12 +10,27 @@ Pass-through endpoints for `/openai` | Logging | ✅ | Works across all integrations | | Streaming | ✅ | Fully supported | -### When to use this? +## Available Endpoints + +### `/openai_passthrough` - Recommended +Dedicated passthrough endpoint that guarantees direct routing to OpenAI without conflicts. + +**Use this for:** +- OpenAI Responses API (`/v1/responses`) +- Any endpoint where you need guaranteed passthrough +- When `/openai` routes are conflicting with LiteLLM's native implementations + +### `/openai` - Legacy +Standard passthrough endpoint that may conflict with LiteLLM's native implementations. + +**Note:** Some endpoints like `/openai/v1/responses` will be routed to LiteLLM's native implementation instead of OpenAI. + +## When to use this? - For 90% of your use cases, you should use the [native LiteLLM OpenAI Integration](https://docs.litellm.ai/docs/providers/openai) (`/chat/completions`, `/embeddings`, `/completions`, `/images`, `/batches`, etc.) -- Use this passthrough to call less popular or newer OpenAI endpoints that LiteLLM doesn't fully support yet, such as `/assistants`, `/threads`, `/vector_stores` +- Use `/openai_passthrough` to call less popular or newer OpenAI endpoints that LiteLLM doesn't fully support yet, such as `/assistants`, `/threads`, `/vector_stores`, `/responses` -Simply replace `https://api.openai.com` with `LITELLM_PROXY_BASE_URL/openai` +Simply replace `https://api.openai.com` with `LITELLM_PROXY_BASE_URL/openai_passthrough` ## Usage Examples @@ -34,7 +49,7 @@ Make sure you do the following: import openai client = openai.OpenAI( - base_url="http://0.0.0.0:4000/openai", # /openai + base_url="http://0.0.0.0:4000/openai_passthrough", # /openai_passthrough api_key="sk-anything" # ) ``` diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d9f824be222..23ef558aeab 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -382,6 +382,7 @@ class LiteLLMRoutes(enum.Enum): "/azure", "/azure_ai", "/openai", + "/openai_passthrough", "/assemblyai", "/eu.assemblyai", "/vllm", diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index c11dcd79bc8..371d0778eb4 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1879,6 +1879,11 @@ async def vertex_proxy_route( ) +@router.api_route( + "/openai_passthrough/{endpoint:path}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH"], + tags=["OpenAI Pass-through", "pass-through"], +) @router.api_route( "/openai/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], @@ -1891,9 +1896,27 @@ async def openai_proxy_route( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ - Simple pass-through for OpenAI. Use this if you want to directly send a request to OpenAI. - - + Pass-through endpoint for OpenAI API calls. + + Available on both routes: + - /openai/{endpoint:path} - Standard OpenAI passthrough route + - /openai_passthrough/{endpoint:path} - Dedicated passthrough route (recommended for Responses API) + + Use /openai_passthrough/* when you need guaranteed passthrough to OpenAI without conflicts + with LiteLLM's native implementations (e.g., for the Responses API at /v1/responses). + + Examples: + Standard route: + - /openai/v1/chat/completions + - /openai/v1/assistants + - /openai/v1/threads + + Dedicated passthrough (for Responses API): + - /openai_passthrough/v1/responses + - /openai_passthrough/v1/responses/{response_id} + - /openai_passthrough/v1/responses/{response_id}/input_items + + [Docs](https://docs.litellm.ai/docs/pass_through/openai_passthrough) """ base_target_url = os.getenv("OPENAI_API_BASE") or "https://api.openai.com/" # Add or update query parameters diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 368e2103590..a0953bf88c7 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -22,6 +22,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( create_pass_through_route, llm_passthrough_factory_proxy_route, milvus_proxy_route, + openai_proxy_route, vertex_discovery_proxy_route, vertex_proxy_route, vllm_proxy_route, @@ -2189,3 +2190,178 @@ class TestMilvusProxyRoute: # Verify that the target URL has correct path create_route_args = mock_create_route.call_args[1] assert "/vectors/search" in create_route_args["target"] + + +class TestOpenAIPassthroughRoute: + """ + Test cases for OpenAI passthrough endpoint (/openai_passthrough) + """ + + @pytest.mark.asyncio + async def test_openai_passthrough_responses_api(self): + """ + Test that /openai_passthrough endpoint correctly handles Responses API calls + This verifies the fix for issue #18865 where /openai/v1/responses was being + routed to LiteLLM's native implementation instead of passthrough + """ + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + openai_proxy_route, + ) + + # Mock request for Responses API + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.headers = {"content-type": "application/json"} + mock_request.query_params = {} + mock_response = MagicMock(spec=Response) + mock_user_api_key_dict = MagicMock() + + with patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials", + return_value="sk-test-key", + ), patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route" + ) as mock_create_route: + mock_endpoint_func = AsyncMock( + return_value={"id": "resp_123", "status": "completed"} + ) + mock_create_route.return_value = mock_endpoint_func + + # Call the route with /v1/responses endpoint + result = await openai_proxy_route( + endpoint="v1/responses", + request=mock_request, + fastapi_response=mock_response, + user_api_key_dict=mock_user_api_key_dict, + ) + + # Verify create_pass_through_route was called with correct target + mock_create_route.assert_called_once() + call_args = mock_create_route.call_args[1] + + # Should route to OpenAI's responses API + assert call_args["target"] == "https://api.openai.com/v1/responses" + assert call_args["endpoint"] == "v1/responses" + + # Verify headers contain API key + assert "authorization" in call_args["custom_headers"] + assert "Bearer sk-test-key" in call_args["custom_headers"]["authorization"] + + # Verify result + assert result == {"id": "resp_123", "status": "completed"} + + @pytest.mark.asyncio + async def test_openai_passthrough_chat_completions(self): + """ + Test that /openai_passthrough works for chat completions + """ + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + openai_proxy_route, + ) + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.headers = {"content-type": "application/json"} + mock_request.query_params = {} + mock_response = MagicMock(spec=Response) + mock_user_api_key_dict = MagicMock() + + with patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials", + return_value="sk-test-key", + ), patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route" + ) as mock_create_route: + mock_endpoint_func = AsyncMock( + return_value={"id": "chatcmpl-123", "choices": []} + ) + mock_create_route.return_value = mock_endpoint_func + + result = await openai_proxy_route( + endpoint="v1/chat/completions", + request=mock_request, + fastapi_response=mock_response, + user_api_key_dict=mock_user_api_key_dict, + ) + + # Verify routing + mock_create_route.assert_called_once() + call_args = mock_create_route.call_args[1] + assert call_args["target"] == "https://api.openai.com/v1/chat/completions" + + # Verify result + assert result == {"id": "chatcmpl-123", "choices": []} + + @pytest.mark.asyncio + async def test_openai_passthrough_missing_api_key(self): + """ + Test that missing OPENAI_API_KEY raises an exception + """ + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + openai_proxy_route, + ) + + mock_request = MagicMock(spec=Request) + mock_response = MagicMock(spec=Response) + mock_user_api_key_dict = MagicMock() + + with patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials", + return_value=None, + ): + with pytest.raises(Exception) as exc_info: + await openai_proxy_route( + endpoint="v1/chat/completions", + request=mock_request, + fastapi_response=mock_response, + user_api_key_dict=mock_user_api_key_dict, + ) + + assert "Required 'OPENAI_API_KEY'" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_openai_passthrough_assistants_api(self): + """ + Test that /openai_passthrough works for Assistants API endpoints + """ + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + openai_proxy_route, + ) + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.headers = {"content-type": "application/json"} + mock_request.query_params = {} + mock_request.url = MagicMock() + mock_request.url.path = "/v1/assistants" + mock_response = MagicMock(spec=Response) + mock_user_api_key_dict = MagicMock() + + with patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials", + return_value="sk-test-key", + ), patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route" + ) as mock_create_route: + mock_endpoint_func = AsyncMock( + return_value={"id": "asst_123", "object": "assistant"} + ) + mock_create_route.return_value = mock_endpoint_func + + result = await openai_proxy_route( + endpoint="v1/assistants", + request=mock_request, + fastapi_response=mock_response, + user_api_key_dict=mock_user_api_key_dict, + ) + + # Verify routing + mock_create_route.assert_called_once() + call_args = mock_create_route.call_args[1] + assert call_args["target"] == "https://api.openai.com/v1/assistants" + + # Verify headers contain API key and OpenAI-Beta header + assert "authorization" in call_args["custom_headers"] + + # Verify result + assert result == {"id": "asst_123", "object": "assistant"}