diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index 1f4a1687e8b..ce03642747c 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -434,7 +434,56 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ ## Getting Reasoning Content in `/chat/completions` -GPT-5 models return reasoning content when called via the Responses API. You can call these models via the `/chat/completions` endpoint by using the `openai/responses/` prefix. +GPT-5 models return reasoning content when called via the Responses API. You can call these models via the `/chat/completions` endpoint in two ways: + +**Option A — per-request prefix:** Use the `openai/responses/` model prefix. + +**Option B — global flag (recommended):** Set `route_all_chat_openai_to_responses = True` to automatically route all OpenAI `/chat/completions` requests through the Responses API, no model prefix needed. + + + + +```python +import litellm + +litellm.route_all_chat_openai_to_responses = True + +response = litellm.completion( + model="gpt-5.4", + messages=[{"role": "user", "content": "What is the capital of France?"}], + reasoning_effort="low", +) +``` + + + + +Set in your proxy config: +```yaml +litellm_settings: + route_all_chat_openai_to_responses: true +``` + +Then call normally — no model prefix needed: +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "gpt-5.4", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "reasoning_effort": "low" +}' +``` + + + + +:::note +`route_all_chat_openai_to_responses` only applies to the `openai` provider. Azure OpenAI is unaffected. You can also set it via env var: `LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES=true`. +::: + +**Option A — per-request prefix:** You can also prefix individual model names with `openai/responses/` to route just that call through the Responses API. diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index cc9090c2de6..e7693953dbe 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -197,6 +197,7 @@ router_settings: | key_generation_settings | object | Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation) | | disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. | | use_chat_completions_url_for_anthropic_messages | boolean | If true, routes OpenAI `/v1/messages` requests through chat/completions instead of the Responses API. Can also be set via env var `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true`. | +| route_all_chat_openai_to_responses | boolean | If true, routes all OpenAI `/chat/completions` requests through the Responses API bridge. Recommended for OpenAI models. Can also be set via env var `LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES=true`. | | disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). | | enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. | | enable_key_alias_format_validation | boolean | If true, validates `key_alias` format on `/key/generate` and `/key/update`. Must be 2-255 chars, start/end with alphanumeric, only allow `a-zA-Z0-9_-/.@`. Default `false`. | @@ -850,6 +851,7 @@ router_settings: | LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM | LITELLM_TOKEN | Access token for LiteLLM integration | LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES | When set to "true", routes OpenAI /v1/messages requests through chat/completions instead of the Responses API for Anthropic models. Can also be set via `litellm_settings.use_chat_completions_url_for_anthropic_messages` +| LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES | When set to "true", routes all OpenAI /chat/completions requests through the Responses API bridge. Recommended for OpenAI models. Can also be set via `litellm_settings.route_all_chat_openai_to_responses` | LITELLM_USER_AGENT | Custom user agent string for LiteLLM API requests. Used for partner telemetry attribution | LITELLM_WORKER_STARTUP_HOOKS | Comma-separated list of `module.path:function_name` callables to run in each worker process during startup. Runs early in the worker lifecycle (before config/DB loading). Useful for re-initializing per-process state like [gflags](https://github.com/google/python-gflags). See [Worker Startup Hooks](/proxy/worker_startup_hooks) for details | LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging diff --git a/litellm/__init__.py b/litellm/__init__.py index e45d926e8db..5e9f1a17a41 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -218,6 +218,9 @@ modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False)) use_chat_completions_url_for_anthropic_messages: bool = bool( os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) ) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API +route_all_chat_openai_to_responses: bool = ( + os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true" +) # When True, routes all OpenAI /chat/completions requests through the Responses API bridge retry = True ### AUTH ### api_key: Optional[str] = None diff --git a/litellm/main.py b/litellm/main.py index eace9c630ba..14dac7a6727 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -939,6 +939,14 @@ def responses_api_bridge_check( reasoning_effort: Optional[Any] = None, ) -> Tuple[dict, str]: model_info: Dict[str, Any] = {} + + # Global flag: route ALL OpenAI chat completions through Responses API. + # Returns early with minimal model_info; callers only inspect the "mode" key. + if litellm.route_all_chat_openai_to_responses and custom_llm_provider == "openai": + model = model.replace("responses/", "") + model_info["mode"] = "responses" + return model_info, model + try: model_info = cast( dict, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 8449620c693..0eb2b5123ff 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -185,8 +185,9 @@ class LiteLLMCompletionResponsesConfig: reasoning_param = responses_api_request.get("reasoning") if reasoning_param: if isinstance(reasoning_param, dict): - # reasoning can be {"effort": "low|medium|high"} - reasoning_effort = reasoning_param.get("effort") + # reasoning can be {"effort": "low|medium|high", "summary": "detailed"} + # Preserve the full dict structure for reasoning_effort + reasoning_effort = reasoning_param elif isinstance(reasoning_param, str): # reasoning could be a string directly reasoning_effort = reasoning_param diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index d19d1d1d756..e3386746fba 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -207,7 +207,7 @@ async def test_url_with_format_param(model, sync_mode, monkeypatch): json_str = json_str.decode("utf-8") print(f"type of json_str: {type(json_str)}") - + # Bedrock models convert URLs to base64, while direct Anthropic models support URLs # bedrock/invoke models use Anthropic messages API which supports URLs if model.startswith("bedrock/invoke/"): @@ -433,7 +433,7 @@ async def test_extra_body_with_fallback( monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") # Flush cache to ensure no stale aiohttp clients are used litellm.in_memory_llm_clients_cache.flush_cache() - + # Set up test parameters model = "openrouter/deepseek/deepseek-chat" messages = [{"role": "user", "content": "Hello, world!"}] @@ -466,8 +466,12 @@ async def test_extra_body_with_fallback( "finish_reason": "stop", } ], - "usage": {"prompt_tokens": 9, "completion_tokens": 12, "total_tokens": 21}, - } + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21, + }, + }, ) response = await litellm.acompletion( @@ -480,8 +484,10 @@ async def test_extra_body_with_fallback( # Verify the response assert response is not None - assert len(respx_mock.calls) > 0, "Mock was not called - check if aiohttp transport is properly disabled" - + assert ( + len(respx_mock.calls) > 0 + ), "Mock was not called - check if aiohttp transport is properly disabled" + # Get the request from the mock request: httpx.Request = respx_mock.calls[0].request request_body = request.read() @@ -523,35 +529,43 @@ async def test_openai_env_base( # Configure respx mock to intercept the request mock_route = respx_mock.post( url__regex=r"http://localhost:12345/v1/chat/completions.*" - ).mock(return_value=httpx.Response( - status_code=200, - json={ - "id": "chatcmpl-123", - "object": "chat.completion", - "created": 1677652288, - "model": model, - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Hello from mocked response!", - }, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 9, "completion_tokens": 12, "total_tokens": 21}, - } - )) + ).mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": model, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello from mocked response!", + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21, + }, + }, + ) + ) try: response = await litellm.acompletion(model=model, messages=messages) - + # verify we had a response assert response.choices[0].message.content == "Hello from mocked response!" - + # Verify the mock was called - assert mock_route.called, "Mock route was not called - request may have bypassed respx" + assert ( + mock_route.called + ), "Mock route was not called - request may have bypassed respx" finally: # Clean up to avoid affecting other tests litellm.disable_aiohttp_transport = False @@ -622,9 +636,9 @@ def test_responses_api_bridge_check_gpt_5_4_pro(): model=model_name, custom_llm_provider="openai", ) - assert model_info.get("mode") == "responses", ( - f"{model_name} should have mode='responses', got '{model_info.get('mode')}'" - ) + assert ( + model_info.get("mode") == "responses" + ), f"{model_name} should have mode='responses', got '{model_info.get('mode')}'" def test_responses_api_bridge_check_gpt_5_4_tools_plus_reasoning_routes_to_responses(): @@ -764,6 +778,49 @@ def test_responses_api_bridge_check_handles_exception(): assert model_info["mode"] == "responses" +def test_responses_api_bridge_check_global_flag_routes_openai(): + """When route_all_chat_openai_to_responses is True, any OpenAI model routes to responses.""" + from litellm.main import responses_api_bridge_check + + with patch.object(litellm, "route_all_chat_openai_to_responses", True): + model_info, model = responses_api_bridge_check( + model="gpt-4o", + custom_llm_provider="openai", + ) + + assert model == "gpt-4o" + assert model_info.get("mode") == "responses" + + +def test_responses_api_bridge_check_global_flag_does_not_affect_azure(): + """route_all_chat_openai_to_responses should not affect Azure models.""" + from litellm.main import responses_api_bridge_check + + with patch.object(litellm, "route_all_chat_openai_to_responses", True): + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 4096} + model_info, model = responses_api_bridge_check( + model="gpt-4o", + custom_llm_provider="azure", + ) + + assert model_info.get("mode") != "responses" + + +def test_responses_api_bridge_check_global_flag_default_false(): + """By default, route_all_chat_openai_to_responses is False and doesn't affect routing.""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 4096} + model_info, model = responses_api_bridge_check( + model="gpt-4o", + custom_llm_provider="openai", + ) + + assert model_info.get("mode") != "responses" + + @pytest.mark.asyncio async def test_async_mock_delay(): """Use asyncio await for mock delay on acompletion""" @@ -1487,7 +1544,7 @@ def test_anthropic_text_disable_url_suffix_env_var(): def test_image_edit_merges_headers_and_extra_headers(): from litellm.images.main import base_llm_http_handler - + combined_headers = { "x-test-header-one": "value-1", "x-test-header-two": "value-2",