scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

feat(responses): honor supported_endpoints /v1/responses opt-in for OpenAI-compatible deployments (#39725)

Browse source 
* feat(responses): honor supported_endpoints /v1/responses opt-in for OpenAI-compatible deployments

custom_openai and other generic OpenAI-compatible deployments have no native
Responses API config, so every /v1/responses call is bridged through
/v1/chat/completions. When model_info.supported_endpoints lists /v1/responses,
resolve OpenAILikeResponsesConfig instead so the request is forwarded to
{api_base}/responses, for streaming, non-streaming and mode: responses
deployments alike. Providers with their own Responses config are unchanged.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(responses): drop deployment supported_endpoints opt-in after cross-provider prompt swap

A prompt manager that moves the request to another provider leaves kwargs['model_info']
describing the original deployment; without this the swapped provider was sent an
OpenAI-like /responses request it does not serve.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(responses): carry prompt-swap deployment metadata as a return value instead of a kwargs marker

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-09-05 11:40:00 -07:00 • committed by GitHub
parent 4df284e16d
commit f66b3ebe0d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 327 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

93
litellm/responses/main.py
View file

@ -2,6 +2,7 @@ import asyncio
import contextvars
from collections.abc import Coroutine, Generator, Iterable, Mapping
from contextlib import contextmanager
from dataclasses import dataclass
from functools import partial
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast
@ -23,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
)
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
from litellm.responses.litellm_completion_transformation.handler import (
LiteLLMCompletionTransformationHandler,
)
@ -403,8 +405,40 @@ def _bridges_to_chat_completions(
return responses_api_provider_config is None or use_chat_completions_api is True
def _deployment_passes_through_responses(model_info: object) -> bool:
"""Whether ``model_info.supported_endpoints`` opts the deployment into native ``{api_base}/responses``."""
if not isinstance(model_info, dict):
return False
supported_endpoints: Final = model_info.get("supported_endpoints")
return isinstance(supported_endpoints, (list, tuple)) and "/v1/responses" in supported_endpoints
def _deployment_model_info_after_prompt_swap(
requested_provider: str | None, resolved_provider: str | None, model_info: object
) -> object:
"""Deployment metadata only describes the upstream while the prompt manager keeps its provider."""
return model_info if resolved_provider == requested_provider else None
@dataclass(frozen=True, slots=True)
class _AsyncPromptManagementOutcome:
merged_optional_params: Mapping[str, object]
deployment_model_info: object
def _resolve_responses_api_provider_config(
model: str, custom_llm_provider: str, model_info: object
) -> BaseResponsesAPIConfig | None:
provider_config: Final = ProviderConfigManager.get_provider_responses_api_config(
model=model, provider=custom_llm_provider
)
if provider_config is not None or not _deployment_passes_through_responses(model_info):
return provider_config
return OpenAILikeResponsesConfig()
def _will_bridge_to_chat_completions(
model: str, custom_llm_provider: str | None, use_chat_completions_api: bool
model: str, custom_llm_provider: str | None, use_chat_completions_api: bool, model_info: object
) -> bool:
"""``_bridges_to_chat_completions`` for callers running before the provider config is resolved.
@ -418,9 +452,7 @@ def _will_bridge_to_chat_completions(
if custom_llm_provider is None:
return True
return _bridges_to_chat_completions(
ProviderConfigManager.get_provider_responses_api_config(
model=normalized_model[0], provider=custom_llm_provider
),
_resolve_responses_api_provider_config(normalized_model[0], custom_llm_provider, model_info),
use_chat_completions_api or normalized_model[1],
)
@ -527,7 +559,10 @@ async def aresponses(
with _prompt_management_sees_a_provisional_message_list(
kwargs,
bridged=_will_bridge_to_chat_completions(
model, custom_llm_provider, bool(kwargs.get("use_chat_completions_api"))
model,
custom_llm_provider,
bool(kwargs.get("use_chat_completions_api")),
kwargs.get("model_info"),
),
):
(
@ -552,6 +587,7 @@ async def aresponses(
merged_input=merged_input,
),
)
requested_provider: Final = custom_llm_provider
if model != original_model:
custom_llm_provider = _resolve_prompt_swapped_provider(
original_model=original_model,
@ -561,7 +597,12 @@ async def aresponses(
prompt_id=prompt_id,
)
kwargs.pop("prompt_id", None)
kwargs["_async_prompt_merged_params"] = merged_optional_params
kwargs["_async_prompt_merged_params"] = _AsyncPromptManagementOutcome(
merged_optional_params=merged_optional_params,
deployment_model_info=_deployment_model_info_after_prompt_swap(
requested_provider, custom_llm_provider, kwargs.get("model_info")
),
)
func: Final = partial(
responses,
@ -666,12 +707,14 @@ def _apply_prompt_management_to_responses_call(
kwargs: dict[str, Any],
local_vars: dict[str, object],
use_chat_completions_api: bool,
) -> tuple[str | ResponseInputParam, str, str | None]:
async_merged: Final[Mapping[str, object] | None] = kwargs.pop("_async_prompt_merged_params", None)
if async_merged is not None:
for key, value in async_merged.items():
) -> tuple[str | ResponseInputParam, str, str | None, object]:
"""Returns the prompt-managed input, model and provider, plus the deployment metadata that still
describes the upstream (``None`` once the prompt manager moved the request to another provider)."""
async_outcome: Final[_AsyncPromptManagementOutcome | None] = kwargs.pop("_async_prompt_merged_params", None)
if async_outcome is not None:
for key, value in async_outcome.merged_optional_params.items():
local_vars[key] = value
return input, model, custom_llm_provider
return input, model, custom_llm_provider, async_outcome.deployment_model_info
prompt_id: Final = cast(str | None, kwargs.get("prompt_id", None))
prompt_variables: Final = cast(dict | None, kwargs.get("prompt_variables", None))
@ -684,7 +727,9 @@ def _apply_prompt_management_to_responses_call(
):
with _prompt_management_sees_a_provisional_message_list(
kwargs,
bridged=_will_bridge_to_chat_completions(model, custom_llm_provider, use_chat_completions_api),
bridged=_will_bridge_to_chat_completions(
model, custom_llm_provider, use_chat_completions_api, kwargs.get("model_info")
),
):
(
model,
@ -710,19 +755,28 @@ def _apply_prompt_management_to_responses_call(
)
local_vars["input"] = input
local_vars["model"] = model
if model != original_model:
custom_llm_provider = _resolve_prompt_swapped_provider(
resolved_provider: Final = (
custom_llm_provider
if model == original_model
else _resolve_prompt_swapped_provider(
original_model=original_model,
swapped_model=model,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
prompt_id=prompt_id,
)
local_vars["custom_llm_provider"] = custom_llm_provider
)
local_vars["custom_llm_provider"] = resolved_provider
for key, value in merged_optional_params.items():
local_vars[key] = value
return (
input,
model,
resolved_provider,
_deployment_model_info_after_prompt_swap(custom_llm_provider, resolved_provider, kwargs.get("model_info")),
)
return input, model, custom_llm_provider
return input, model, custom_llm_provider, kwargs.get("model_info")
# Opt-in via model id (mirrors the `responses/` prefix pattern on chat completions).
@ -1052,7 +1106,7 @@ def responses(
)
local_vars["custom_llm_provider"] = custom_llm_provider
input, model, custom_llm_provider = _apply_prompt_management_to_responses_call(
input, model, custom_llm_provider, deployment_model_info = _apply_prompt_management_to_responses_call(
input=input,
model=model,
custom_llm_provider=custom_llm_provider,
@ -1123,9 +1177,8 @@ def responses(
if custom_llm_provider is None:
responses_api_provider_config = None
else:
responses_api_provider_config = ProviderConfigManager.get_provider_responses_api_config(
model=model,
provider=custom_llm_provider,
responses_api_provider_config = _resolve_responses_api_provider_config(
model, custom_llm_provider, deployment_model_info
)
local_vars.update(kwargs)

254
tests/test_litellm/responses/test_responses_supported_endpoints_passthrough.py Normal file
View file

@ -0,0 +1,254 @@
"""
A deployment with `model_info.supported_endpoints` containing `/v1/responses` forwards
`/v1/responses` natively to `{api_base}/responses`. Without it, generic OpenAI-compatible
providers such as `custom_openai` keep bridging through `/v1/chat/completions`.
"""
import json
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
import respx
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
from litellm.responses.main import _resolve_responses_api_provider_config
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import ModelResponse
API_BASE = "https://backend.example/v1"
RESPONSES_URL = f"{API_BASE}/responses"
CHAT_URL = f"{API_BASE}/chat/completions"
OPT_IN = {"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]}
RESPONSES_BODY = {
"id": "resp_native",
"object": "response",
"created_at": 1741476542,
"status": "completed",
"model": "my-model",
"output": [
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "native", "annotations": []}],
}
],
"parallel_tool_calls": True,
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
}
CHAT_BODY = {
"id": "chatcmpl_bridged",
"object": "chat.completion",
"created": 1741476542,
"model": "my-model",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "bridged"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
SSE_BODY = (
"event: response.created\n"
f"data: {json.dumps({'type': 'response.created', 'response': RESPONSES_BODY})}\n\n"
"event: response.completed\n"
f"data: {json.dumps({'type': 'response.completed', 'response': RESPONSES_BODY})}\n\n"
)
def _mock_backend(router: respx.MockRouter) -> tuple[respx.Route, respx.Route]:
responses_route = router.post(RESPONSES_URL).mock(return_value=httpx.Response(200, json=RESPONSES_BODY))
chat_route = router.post(CHAT_URL).mock(return_value=httpx.Response(200, json=CHAT_BODY))
return responses_route, chat_route
SWAPPED_MODEL = "deepseek/deepseek-chat"
SWAPPED_API_BASE = "https://api.deepseek.com/beta"
def _prompt_manager_swapping_to(model: str) -> MagicMock:
"""A logging object whose prompt hook rewrites the request's model, as a prompt manager does."""
prompt_return = (model, [{"role": "user", "content": "hi"}], {})
logging_obj = MagicMock()
logging_obj.__class__ = LiteLLMLoggingObj
logging_obj.should_run_prompt_management_hooks.return_value = True
logging_obj.get_chat_completion_prompt.return_value = prompt_return
logging_obj.async_get_chat_completion_prompt = AsyncMock(return_value=prompt_return)
logging_obj.model_call_details = {}
return logging_obj
def _mock_swap_targets(router: respx.MockRouter, monkeypatch) -> tuple[respx.Route, respx.Route]:
"""The swapped provider's chat endpoint, plus the `/responses` it does not serve but a stale
opt-in would send to."""
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-deepseek")
swapped_chat_route = router.post(f"{SWAPPED_API_BASE}/chat/completions").mock(
return_value=httpx.Response(200, json=CHAT_BODY)
)
stale_responses_route = router.post(f"{SWAPPED_API_BASE}/responses").mock(
return_value=httpx.Response(200, json=RESPONSES_BODY)
)
return swapped_chat_route, stale_responses_route
@pytest.fixture(autouse=True)
def _respx_interceptable_httpx_client(monkeypatch):
monkeypatch.setattr(litellm, "num_retries", 0)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
yield
litellm.in_memory_llm_clients_cache.flush_cache()
@pytest.mark.parametrize(
"model_info, expected_type",
[
(OPT_IN, OpenAILikeResponsesConfig),
({"supported_endpoints": ["/v1/chat/completions"]}, type(None)),
({}, type(None)),
(None, type(None)),
("/v1/responses", type(None)),
],
)
def test_resolver_opt_in_gates_openai_like_config(model_info, expected_type):
config = _resolve_responses_api_provider_config("my-model", "custom_openai", model_info)
assert type(config) is expected_type
def test_resolver_keeps_native_provider_config():
"""`openai/` already routes /v1/responses natively; the opt-in must not swap its config."""
config = _resolve_responses_api_provider_config("gpt-4.1", "openai", OPT_IN)
assert type(config) is OpenAIResponsesAPIConfig
@respx.mock
async def test_opt_in_forwards_responses_natively():
responses_route, chat_route = _mock_backend(respx.mock)
result = await litellm.aresponses(
model="custom_openai/my-model",
input="hi",
api_base=API_BASE,
api_key="sk-backend",
model_info=OPT_IN,
)
assert responses_route.call_count == 1
assert chat_route.call_count == 0
request = responses_route.calls.last.request
assert request.headers["authorization"] == "Bearer sk-backend"
assert json.loads(request.content)["input"] == "hi"
assert isinstance(result, ResponsesAPIResponse)
assert result.output[0].content[0].text == "native"
@respx.mock
async def test_opt_in_forwards_streaming_responses_natively(monkeypatch):
"""The router registers each deployment in `litellm.model_cost`; an unregistered model is
treated as non-streaming and would be faked, so mirror that registration here."""
monkeypatch.setitem(litellm.model_cost, "custom_openai/my-model", {"litellm_provider": "custom_openai"})
responses_route = respx.post(RESPONSES_URL).mock(
return_value=httpx.Response(200, text=SSE_BODY, headers={"content-type": "text/event-stream"})
)
chat_route = respx.post(CHAT_URL).mock(return_value=httpx.Response(200, json=CHAT_BODY))
stream = await litellm.aresponses(
model="custom_openai/my-model",
input="hi",
stream=True,
api_base=API_BASE,
api_key="sk-backend",
model_info=OPT_IN,
)
events = [event async for event in stream]
assert responses_route.call_count == 1
assert chat_route.call_count == 0
assert json.loads(responses_route.calls.last.request.content)["stream"] is True
assert [event.type for event in events] == ["response.created", "response.completed"]
@respx.mock
async def test_without_opt_in_still_bridges_through_chat_completions():
responses_route, chat_route = _mock_backend(respx.mock)
result = await litellm.aresponses(
model="custom_openai/my-model",
input="hi",
api_base=API_BASE,
api_key="sk-backend",
model_info={"supported_endpoints": ["/v1/chat/completions"]},
)
assert chat_route.call_count == 1
assert responses_route.call_count == 0
assert isinstance(result, ResponsesAPIResponse)
assert result.output[0].content[0].text == "bridged"
@respx.mock
async def test_prompt_swap_to_other_provider_drops_deployment_opt_in(monkeypatch):
"""When a prompt manager moves the request to another provider, the original deployment's
`supported_endpoints` no longer describes the upstream, so the swapped provider bridges."""
swapped_chat_route, stale_responses_route = _mock_swap_targets(respx.mock, monkeypatch)
result = await litellm.aresponses(
model="custom_openai/my-model",
input="hi",
prompt_id="p1",
litellm_logging_obj=_prompt_manager_swapping_to(SWAPPED_MODEL),
model_info=OPT_IN,
)
assert swapped_chat_route.call_count == 1
assert stale_responses_route.call_count == 0
assert isinstance(result, ResponsesAPIResponse)
assert result.output[0].content[0].text == "bridged"
@respx.mock
def test_sync_prompt_swap_to_other_provider_drops_deployment_opt_in(monkeypatch):
swapped_chat_route, stale_responses_route = _mock_swap_targets(respx.mock, monkeypatch)
result = litellm.responses(
model="custom_openai/my-model",
input="hi",
prompt_id="p1",
litellm_logging_obj=_prompt_manager_swapping_to(SWAPPED_MODEL),
model_info=OPT_IN,
)
assert swapped_chat_route.call_count == 1
assert stale_responses_route.call_count == 0
assert isinstance(result, ResponsesAPIResponse)
assert result.output[0].content[0].text == "bridged"
@respx.mock
async def test_mode_responses_chat_completion_reaches_native_responses(monkeypatch):
"""A `mode: responses` deployment bridges chat completions into the Responses API; with
the opt-in that inner call must reach `{api_base}/responses` instead of bouncing back
to `/chat/completions`."""
responses_route, chat_route = _mock_backend(respx.mock)
monkeypatch.setitem(
litellm.model_cost,
"custom_openai/my-model",
{"mode": "responses", "litellm_provider": "custom_openai"},
)
result = await litellm.acompletion(
model="custom_openai/my-model",
messages=[{"role": "user", "content": "hi"}],
api_base=API_BASE,
api_key="sk-backend",
model_info={"mode": "responses", **OPT_IN},
)
assert responses_route.call_count == 1
assert chat_route.call_count == 0
assert isinstance(result, ModelResponse)
assert result.choices[0].message.content == "native"