From 724ad7cbebe3b79c7501461ebf4b86851298ad8d Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 1 Apr 2026 17:31:19 +0530 Subject: [PATCH 1/3] feat(openai): add route_all_chat_openai_to_responses global flag Adds `litellm.route_all_chat_openai_to_responses` (env: `LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES`) to route all OpenAI /chat/completions requests through the Responses API bridge. Also fixes reasoning param dict passthrough in completion transformation. Co-Authored-By: Claude Sonnet 4.6 --- docs/my-website/docs/proxy/config_settings.md | 2 + litellm/__init__.py | 3 + litellm/main.py | 7 + .../transformation.py | 11 +- tests/test_litellm/test_main.py | 123 +++++++++++++----- 5 files changed, 108 insertions(+), 38 deletions(-) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index cc9090c2de6..e7693953dbe 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -197,6 +197,7 @@ router_settings: | key_generation_settings | object | Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation) | | disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. | | use_chat_completions_url_for_anthropic_messages | boolean | If true, routes OpenAI `/v1/messages` requests through chat/completions instead of the Responses API. Can also be set via env var `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true`. | +| route_all_chat_openai_to_responses | boolean | If true, routes all OpenAI `/chat/completions` requests through the Responses API bridge. Recommended for OpenAI models. Can also be set via env var `LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES=true`. | | disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). | | enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. | | enable_key_alias_format_validation | boolean | If true, validates `key_alias` format on `/key/generate` and `/key/update`. Must be 2-255 chars, start/end with alphanumeric, only allow `a-zA-Z0-9_-/.@`. Default `false`. | @@ -850,6 +851,7 @@ router_settings: | LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM | LITELLM_TOKEN | Access token for LiteLLM integration | LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES | When set to "true", routes OpenAI /v1/messages requests through chat/completions instead of the Responses API for Anthropic models. Can also be set via `litellm_settings.use_chat_completions_url_for_anthropic_messages` +| LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES | When set to "true", routes all OpenAI /chat/completions requests through the Responses API bridge. Recommended for OpenAI models. Can also be set via `litellm_settings.route_all_chat_openai_to_responses` | LITELLM_USER_AGENT | Custom user agent string for LiteLLM API requests. Used for partner telemetry attribution | LITELLM_WORKER_STARTUP_HOOKS | Comma-separated list of `module.path:function_name` callables to run in each worker process during startup. Runs early in the worker lifecycle (before config/DB loading). Useful for re-initializing per-process state like [gflags](https://github.com/google/python-gflags). See [Worker Startup Hooks](/proxy/worker_startup_hooks) for details | LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging diff --git a/litellm/__init__.py b/litellm/__init__.py index e45d926e8db..16451fd1f60 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -218,6 +218,9 @@ modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False)) use_chat_completions_url_for_anthropic_messages: bool = bool( os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) ) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API +route_all_chat_openai_to_responses: bool = bool( + os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", False) +) # When True, routes all OpenAI /chat/completions requests through the Responses API bridge retry = True ### AUTH ### api_key: Optional[str] = None diff --git a/litellm/main.py b/litellm/main.py index eace9c630ba..e81a179c61b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -939,6 +939,13 @@ def responses_api_bridge_check( reasoning_effort: Optional[Any] = None, ) -> Tuple[dict, str]: model_info: Dict[str, Any] = {} + + # Global flag: route ALL OpenAI chat completions through Responses API + if litellm.route_all_chat_openai_to_responses and custom_llm_provider == "openai": + model = model.replace("responses/", "") + model_info["mode"] = "responses" + return model_info, model + try: model_info = cast( dict, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 9075373f1cf..debd9f77b7e 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -185,8 +185,9 @@ class LiteLLMCompletionResponsesConfig: reasoning_param = responses_api_request.get("reasoning") if reasoning_param: if isinstance(reasoning_param, dict): - # reasoning can be {"effort": "low|medium|high"} - reasoning_effort = reasoning_param.get("effort") + # reasoning can be {"effort": "low|medium|high", "summary": "detailed"} + # Preserve the full dict structure for reasoning_effort + reasoning_effort = reasoning_param elif isinstance(reasoning_param, str): # reasoning could be a string directly reasoning_effort = reasoning_param @@ -2123,9 +2124,9 @@ class LiteLLMCompletionResponsesConfig: hasattr(completion_details, "reasoning_tokens") and completion_details.reasoning_tokens is not None ): - output_details_dict["reasoning_tokens"] = ( - completion_details.reasoning_tokens - ) + output_details_dict[ + "reasoning_tokens" + ] = completion_details.reasoning_tokens else: output_details_dict["reasoning_tokens"] = 0 diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index ce5873f5063..217f9328dc7 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -207,7 +207,7 @@ async def test_url_with_format_param(model, sync_mode, monkeypatch): json_str = json_str.decode("utf-8") print(f"type of json_str: {type(json_str)}") - + # Bedrock models convert URLs to base64, while direct Anthropic models support URLs # bedrock/invoke models use Anthropic messages API which supports URLs if model.startswith("bedrock/invoke/"): @@ -433,7 +433,7 @@ async def test_extra_body_with_fallback( monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") # Flush cache to ensure no stale aiohttp clients are used litellm.in_memory_llm_clients_cache.flush_cache() - + # Set up test parameters model = "openrouter/deepseek/deepseek-chat" messages = [{"role": "user", "content": "Hello, world!"}] @@ -466,8 +466,12 @@ async def test_extra_body_with_fallback( "finish_reason": "stop", } ], - "usage": {"prompt_tokens": 9, "completion_tokens": 12, "total_tokens": 21}, - } + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21, + }, + }, ) response = await litellm.acompletion( @@ -480,8 +484,10 @@ async def test_extra_body_with_fallback( # Verify the response assert response is not None - assert len(respx_mock.calls) > 0, "Mock was not called - check if aiohttp transport is properly disabled" - + assert ( + len(respx_mock.calls) > 0 + ), "Mock was not called - check if aiohttp transport is properly disabled" + # Get the request from the mock request: httpx.Request = respx_mock.calls[0].request request_body = request.read() @@ -523,35 +529,43 @@ async def test_openai_env_base( # Configure respx mock to intercept the request mock_route = respx_mock.post( url__regex=r"http://localhost:12345/v1/chat/completions.*" - ).mock(return_value=httpx.Response( - status_code=200, - json={ - "id": "chatcmpl-123", - "object": "chat.completion", - "created": 1677652288, - "model": model, - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Hello from mocked response!", - }, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 9, "completion_tokens": 12, "total_tokens": 21}, - } - )) + ).mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": model, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello from mocked response!", + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21, + }, + }, + ) + ) try: response = await litellm.acompletion(model=model, messages=messages) - + # verify we had a response assert response.choices[0].message.content == "Hello from mocked response!" - + # Verify the mock was called - assert mock_route.called, "Mock route was not called - request may have bypassed respx" + assert ( + mock_route.called + ), "Mock route was not called - request may have bypassed respx" finally: # Clean up to avoid affecting other tests litellm.disable_aiohttp_transport = False @@ -622,9 +636,9 @@ def test_responses_api_bridge_check_gpt_5_4_pro(): model=model_name, custom_llm_provider="openai", ) - assert model_info.get("mode") == "responses", ( - f"{model_name} should have mode='responses', got '{model_info.get('mode')}'" - ) + assert ( + model_info.get("mode") == "responses" + ), f"{model_name} should have mode='responses', got '{model_info.get('mode')}'" def test_responses_api_bridge_check_gpt_5_4_tools_plus_reasoning_routes_to_responses(): @@ -764,6 +778,49 @@ def test_responses_api_bridge_check_handles_exception(): assert model_info["mode"] == "responses" +def test_responses_api_bridge_check_global_flag_routes_openai(): + """When route_all_chat_openai_to_responses is True, any OpenAI model routes to responses.""" + from litellm.main import responses_api_bridge_check + + with patch.object(litellm, "route_all_chat_openai_to_responses", True): + model_info, model = responses_api_bridge_check( + model="gpt-4o", + custom_llm_provider="openai", + ) + + assert model == "gpt-4o" + assert model_info.get("mode") == "responses" + + +def test_responses_api_bridge_check_global_flag_does_not_affect_azure(): + """route_all_chat_openai_to_responses should not affect Azure models.""" + from litellm.main import responses_api_bridge_check + + with patch.object(litellm, "route_all_chat_openai_to_responses", True): + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 4096} + model_info, model = responses_api_bridge_check( + model="gpt-4o", + custom_llm_provider="azure", + ) + + assert model_info.get("mode") != "responses" + + +def test_responses_api_bridge_check_global_flag_default_false(): + """By default, route_all_chat_openai_to_responses is False and doesn't affect routing.""" + from litellm.main import responses_api_bridge_check + + with patch("litellm.main._get_model_info_helper") as mock_get_model_info: + mock_get_model_info.return_value = {"max_tokens": 4096} + model_info, model = responses_api_bridge_check( + model="gpt-4o", + custom_llm_provider="openai", + ) + + assert model_info.get("mode") != "responses" + + @pytest.mark.asyncio async def test_async_mock_delay(): """Use asyncio await for mock delay on acompletion""" @@ -1487,7 +1544,7 @@ def test_anthropic_text_disable_url_suffix_env_var(): def test_image_edit_merges_headers_and_extra_headers(): from litellm.images.main import base_llm_http_handler - + combined_headers = { "x-test-header-one": "value-1", "x-test-header-two": "value-2", From bbcfabe08a572e3aa788ef1044796d49ab5549b1 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 1 Apr 2026 17:47:44 +0530 Subject: [PATCH 2/3] fix(openai): fix env var bool parsing and add responses API docs - Use .lower() == "true" for LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES to avoid bool("False") == True bug - Add clarifying comment on early return in responses_api_bridge_check - Document route_all_chat_openai_to_responses flag in openai/responses_api.md Co-Authored-By: Claude Sonnet 4.6 --- .../docs/providers/openai/responses_api.md | 39 +++++++++++++++++++ litellm/__init__.py | 4 +- litellm/main.py | 3 +- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/providers/openai/responses_api.md b/docs/my-website/docs/providers/openai/responses_api.md index 0d6b9013ac8..31f21ba52a7 100644 --- a/docs/my-website/docs/providers/openai/responses_api.md +++ b/docs/my-website/docs/providers/openai/responses_api.md @@ -923,6 +923,45 @@ curl http://localhost:4000/v1/chat/completions \ +### Route all OpenAI chat completions through the Responses API (recommended) + +Instead of prefixing each model with `openai/responses/`, you can enable a global flag to automatically route **all** `/chat/completions` requests for OpenAI models through the Responses API bridge. This is the recommended approach for OpenAI models. + + + + +```python showLineNumbers title="Global flag - route all OpenAI completions via Responses API" +import litellm + +litellm.route_all_chat_openai_to_responses = True + +response = litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Hello!"}], +) +``` + + + + +```yaml showLineNumbers title="proxy_config.yaml" +litellm_settings: + route_all_chat_openai_to_responses: true +``` + +Or set via environment variable: + +```bash +LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES=true +``` + + + + +:::note +This flag only applies to the `openai` provider. Azure OpenAI and other providers are unaffected. +::: + ## Free-form Function Calling diff --git a/litellm/__init__.py b/litellm/__init__.py index 16451fd1f60..5e9f1a17a41 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -218,8 +218,8 @@ modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False)) use_chat_completions_url_for_anthropic_messages: bool = bool( os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) ) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API -route_all_chat_openai_to_responses: bool = bool( - os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", False) +route_all_chat_openai_to_responses: bool = ( + os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true" ) # When True, routes all OpenAI /chat/completions requests through the Responses API bridge retry = True ### AUTH ### diff --git a/litellm/main.py b/litellm/main.py index e81a179c61b..14dac7a6727 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -940,7 +940,8 @@ def responses_api_bridge_check( ) -> Tuple[dict, str]: model_info: Dict[str, Any] = {} - # Global flag: route ALL OpenAI chat completions through Responses API + # Global flag: route ALL OpenAI chat completions through Responses API. + # Returns early with minimal model_info; callers only inspect the "mode" key. if litellm.route_all_chat_openai_to_responses and custom_llm_provider == "openai": model = model.replace("responses/", "") model_info["mode"] = "responses" From aabb543f58b2a71f83cd810466939dd60d345a2b Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 1 Apr 2026 17:50:48 +0530 Subject: [PATCH 3/3] docs(openai): move chat-to-responses flag docs to openai.md completions section - Add route_all_chat_openai_to_responses global flag docs under 'Getting Reasoning Content in /chat/completions' in openai.md with SDK and proxy examples using gpt-5.4 - Remove the section from responses_api.md (wrong location) Co-Authored-By: Claude Sonnet 4.6 --- docs/my-website/docs/providers/openai.md | 51 ++++++++++++++++++- .../docs/providers/openai/responses_api.md | 39 -------------- 2 files changed, 50 insertions(+), 40 deletions(-) diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index 1f4a1687e8b..ce03642747c 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -434,7 +434,56 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ ## Getting Reasoning Content in `/chat/completions` -GPT-5 models return reasoning content when called via the Responses API. You can call these models via the `/chat/completions` endpoint by using the `openai/responses/` prefix. +GPT-5 models return reasoning content when called via the Responses API. You can call these models via the `/chat/completions` endpoint in two ways: + +**Option A — per-request prefix:** Use the `openai/responses/` model prefix. + +**Option B — global flag (recommended):** Set `route_all_chat_openai_to_responses = True` to automatically route all OpenAI `/chat/completions` requests through the Responses API, no model prefix needed. + + + + +```python +import litellm + +litellm.route_all_chat_openai_to_responses = True + +response = litellm.completion( + model="gpt-5.4", + messages=[{"role": "user", "content": "What is the capital of France?"}], + reasoning_effort="low", +) +``` + + + + +Set in your proxy config: +```yaml +litellm_settings: + route_all_chat_openai_to_responses: true +``` + +Then call normally — no model prefix needed: +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "gpt-5.4", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "reasoning_effort": "low" +}' +``` + + + + +:::note +`route_all_chat_openai_to_responses` only applies to the `openai` provider. Azure OpenAI is unaffected. You can also set it via env var: `LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES=true`. +::: + +**Option A — per-request prefix:** You can also prefix individual model names with `openai/responses/` to route just that call through the Responses API. diff --git a/docs/my-website/docs/providers/openai/responses_api.md b/docs/my-website/docs/providers/openai/responses_api.md index 31f21ba52a7..0d6b9013ac8 100644 --- a/docs/my-website/docs/providers/openai/responses_api.md +++ b/docs/my-website/docs/providers/openai/responses_api.md @@ -923,45 +923,6 @@ curl http://localhost:4000/v1/chat/completions \ -### Route all OpenAI chat completions through the Responses API (recommended) - -Instead of prefixing each model with `openai/responses/`, you can enable a global flag to automatically route **all** `/chat/completions` requests for OpenAI models through the Responses API bridge. This is the recommended approach for OpenAI models. - - - - -```python showLineNumbers title="Global flag - route all OpenAI completions via Responses API" -import litellm - -litellm.route_all_chat_openai_to_responses = True - -response = litellm.completion( - model="gpt-4o", - messages=[{"role": "user", "content": "Hello!"}], -) -``` - - - - -```yaml showLineNumbers title="proxy_config.yaml" -litellm_settings: - route_all_chat_openai_to_responses: true -``` - -Or set via environment variable: - -```bash -LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES=true -``` - - - - -:::note -This flag only applies to the `openai` provider. Azure OpenAI and other providers are unaffected. -::: - ## Free-form Function Calling