diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index 9b1297dbd75..0152931ebc3 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -1600,7 +1600,8 @@ class LiteLLMAnthropicMessagesAdapter: else translated_finish_reason ) # extract usage - usage: Final[Usage] = getattr(response, "usage") + # A provider reply with no usage block leaves the response without one; it reads as zero tokens. + usage: Final[Usage] = getattr(response, "usage", None) or Usage() message_usage: Final = self.translate_openai_usage_to_anthropic_usage(usage) polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None anthropic_usage: Final[AnthropicUsage] = ( diff --git a/litellm/main.py b/litellm/main.py index 873aef483bf..ac0a2962259 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5443,14 +5443,9 @@ def completion( ######### unpacking kwargs ##################### args: Final = _locals_snapshot(locals()) - # Set by the responses->completion fallback so completion() does not bridge - # back to the Responses API: that round-trip mutually recurses forever for a - # model whose model_cost mode is "responses" but whose provider has no - # Responses API config (get_provider_responses_api_config -> None). - skip_responses_api_bridge: Final = kwargs.pop("_skip_responses_api_bridge", False) - control_options: Final = _resolve_control_options(kwargs, model) skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False) + skip_responses_api_bridge: Final = kwargs.pop("_skip_responses_api_bridge", False) is True if not skip_mcp_handler and tools and may_name_litellm_mcp_gateway(tools): # pyright: ignore[reportUnknownArgumentType] # untyped tools list from litellm.responses.mcp.chat_completions_handler import acompletion_with_mcp @@ -5638,6 +5633,23 @@ def completion( ) +def _bridges_to_responses_api( + responses_api_model_info: Mapping[str, object], + model: str, + custom_llm_provider: str, + model_info: object, + api_base: str | None, +) -> bool: + if responses_api_model_info.get("mode") != "responses": + return False + from litellm.responses.main import resolve_responses_api_provider_config + + return isinstance( + plan_chat_route(resolve_responses_api_provider_config(model, custom_llm_provider, model_info, api_base)), + NativeResponses, + ) + + def execute_chat( *, model: str, @@ -5645,7 +5657,6 @@ def execute_chat( kwargs: dict, # mutable-ok: provider steps pop request-scoped keys, as in completion() non_default_params: dict[str, object], # mutable-ok: seeded in place before the provider call control_options: ControlOptions, - skip_responses_api_bridge: bool, prompt_id: str | None, prompt_variables: dict | None, # mutable-ok: forwarded to get_litellm_params as is timeout: float | str | httpx.Timeout | openai.Timeout | None = None, @@ -5689,7 +5700,9 @@ def execute_chat( api_key: str | None = None, thinking: AnthropicThinkingParam | None = None, shared_session: Optional["ClientSession"] = None, + skip_responses_api_bridge: bool = False, ) -> ModelResponse | CustomStreamWrapper: + """``skip_responses_api_bridge`` is set by the Responses to chat bridge so a ``mode: responses`` deployment stays on chat.""" api_base = kwargs.get("api_base", None) mock_response: Final[MOCK_RESPONSE_TYPE | None] = kwargs.get("mock_response", None) mock_tool_calls: Final = kwargs.get("mock_tool_calls", None) @@ -5913,8 +5926,9 @@ def execute_chat( if dynamic_api_key is not None: api_key = dynamic_api_key # check if user passed in any of the OpenAI optional params - bridges_to_responses_api: Final = ( - responses_api_model_info.get("mode") == "responses" and not skip_responses_api_bridge + deployment_model_info: Final = cast(object, kwargs.get("model_info")) # cast-ok: untyped request kwargs + bridges_to_responses_api: Final = not skip_responses_api_bridge and _bridges_to_responses_api( + responses_api_model_info, model, custom_llm_provider, deployment_model_info, api_base ) allowed_openai_params: Final[list[str] | None] = ( [*(kwargs.get("allowed_openai_params") or []), "reasoning_effort"] @@ -6188,9 +6202,8 @@ def execute_chat( # detection when the deployment name differs from the model name. _azure_detection_model: Final = base_model or model - if isinstance( - plan_chat_route(responses_api_model_info.get("mode") == "responses", skip_responses_api_bridge is True), - NativeResponses, + if not skip_responses_api_bridge and _bridges_to_responses_api( + responses_api_model_info, model, custom_llm_provider, deployment_model_info, api_base ): from litellm.completion_extras import responses_api_bridge @@ -6500,12 +6513,13 @@ def execute_chat( _PLANNED_CHAT_ARGS: Final = frozenset( - ("kwargs", "non_default_params", "control_options", "skip_responses_api_bridge", "prompt_id", "prompt_variables") + ("kwargs", "non_default_params", "control_options", "prompt_id", "prompt_variables") ) _EXECUTE_CHAT_CODE: Final = execute_chat.__code__ _EXECUTE_CHAT_PARAMS: Final = ( frozenset(_EXECUTE_CHAT_CODE.co_varnames[: _EXECUTE_CHAT_CODE.co_kwonlyargcount]) - _PLANNED_CHAT_ARGS ) +_SKIP_RESPONSES_API_BRIDGE_KWARG: Final = "_skip_responses_api_bridge" _CHAT_SURFACE_KWARGS: Final = ( "fallbacks", "model_list", @@ -6517,7 +6531,6 @@ _CHAT_SURFACE_KWARGS: Final = ( "mock_delay", "mock_timeout", ) -_SKIP_RESPONSES_API_BRIDGE: Final = "_skip_responses_api_bridge" _TEXT_COMPLETION_PROVIDERS: Final = frozenset( ("text-completion-openai", "text-completion-codestral", "text-completion-inception") ) @@ -6531,7 +6544,6 @@ class PlannedChat: kwargs: Mapping[str, object] non_default_params: Mapping[str, object] control_options: ControlOptions - skip_responses_api_bridge: bool def _optional_str(value: object) -> str | None: @@ -6604,7 +6616,7 @@ def plan_chat(request: Mapping[str, object], logging_obj: LiteLLMLoggingObj) -> kwargs: Final = { key: value for key, value in request.items() - if key not in _EXECUTE_CHAT_PARAMS and key != _SKIP_RESPONSES_API_BRIDGE + if key not in _EXECUTE_CHAT_PARAMS and key != _SKIP_RESPONSES_API_BRIDGE_KWARG } tools: Final = validate_and_fix_openai_tools(tools=_object_list(request.get("tools"))) messages: Final = get_completion_messages( @@ -6640,12 +6652,12 @@ def plan_chat(request: Mapping[str, object], logging_obj: LiteLLMLoggingObj) -> "tool_choice": tool_choice, "stop": validate_openai_optional_params(stop=_stop(request.get("stop"))), "thinking": validate_and_fix_thinking_param(thinking=_thinking(request.get("thinking"))), + "skip_responses_api_bridge": request.get(_SKIP_RESPONSES_API_BRIDGE_KWARG) is True, } ), kwargs=MappingProxyType(kwargs), non_default_params=MappingProxyType(non_default_params), control_options=_resolve_control_options(kwargs, model), - skip_responses_api_bridge=request.get(_SKIP_RESPONSES_API_BRIDGE) is True, ) @@ -6655,7 +6667,6 @@ def _run_planned_chat(plan: PlannedChat, is_async: bool) -> ModelResponse | Cust kwargs={**plan.kwargs, "acompletion": is_async}, non_default_params=dict(plan.non_default_params), control_options=plan.control_options, - skip_responses_api_bridge=plan.skip_responses_api_bridge, prompt_id=None, prompt_variables=None, ) diff --git a/litellm/native/route_plan.py b/litellm/native/route_plan.py index 00690543db2..07f7f486991 100644 --- a/litellm/native/route_plan.py +++ b/litellm/native/route_plan.py @@ -155,8 +155,6 @@ def plan_responses_route( return NativeResponses() -def plan_chat_route(responses_mode_model: bool, skip_responses_api_bridge: bool) -> NativeChat | NativeResponses: - """``responses_mode_model`` is whether ``responses_api_bridge_check`` resolved the deployment to mode ``responses``.""" - if responses_mode_model and not skip_responses_api_bridge: - return NativeResponses() - return NativeChat() +def plan_chat_route(responses_api_provider_config: BaseResponsesAPIConfig | None) -> NativeChat | NativeResponses: + """For a deployment ``responses_api_bridge_check`` resolved to mode ``responses``.""" + return NativeChat() if responses_api_provider_config is None else NativeResponses() diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 658aa009683..0a973bfeef6 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -66,6 +66,14 @@ def test_translate_openai_response_to_anthropic_empty_choices() -> None: assert result["usage"]["input_tokens"] == 10 +def test_translate_openai_response_to_anthropic_without_usage_reports_zero_tokens() -> None: + response: Final = ModelResponse(choices=[Choices(message=Message(content="x"), finish_reason="stop")]) + + result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) + + assert (result["usage"]["input_tokens"], result["usage"]["output_tokens"]) == (0, 0) + + @pytest.mark.parametrize("text,count,expected_stop", [ ("", 1, "compaction"), (None, 1, "compaction"), ("Answer", 1, "max_tokens"), (" ", 1, "max_tokens"), ("", 2, "max_tokens"), ("", 0, "max_tokens"), diff --git a/tests/unit/native/test_chat_plan.py b/tests/unit/native/test_chat_plan.py index 68958d374a6..8d25a5b61cd 100644 --- a/tests/unit/native/test_chat_plan.py +++ b/tests/unit/native/test_chat_plan.py @@ -1,3 +1,4 @@ +import json from typing import Final import httpx @@ -93,3 +94,44 @@ async def test_a_provider_failure_on_the_messages_to_chat_bridge_is_mapped(respx await _router({"model": "groq/llama-3.1-8b-instant"}, {"mode": "responses"}).aanthropic_messages( model="m", max_tokens=32, messages=[{"role": "user", "content": "hi"}] ) + + +@pytest.mark.asyncio +async def test_a_responses_request_on_a_chat_api_deployment_stays_on_chat_despite_mode_responses( + respx_mock: respx.MockRouter, +): + chat: Final = respx_mock.post(f"{API_BASE}/chat/completions").mock( + return_value=httpx.Response(200, json=_CHAT_REPLY) + ) + + await _router({"model": "openai/gpt-4.1", "use_chat_completions_api": True}, {"mode": "responses"}).aresponses( + model="m", input="hi" + ) + + assert chat.call_count == 1 + + +@pytest.mark.asyncio +async def test_the_skip_bridge_flag_is_never_sent_to_the_provider(respx_mock: respx.MockRouter): + chat: Final = respx_mock.post(f"{API_BASE}/chat/completions").mock( + return_value=httpx.Response(200, json=_CHAT_REPLY) + ) + + await _router({"model": "openai/gpt-4.1"}).acompletion( + model="m", messages=[{"role": "user", "content": "hi"}], _skip_responses_api_bridge=True + ) + + assert "_skip_responses_api_bridge" not in json.loads(chat.calls.last.request.content) + + +@pytest.mark.asyncio +async def test_the_skip_bridge_flag_keeps_a_mode_responses_deployment_on_chat(respx_mock: respx.MockRouter): + chat: Final = respx_mock.post(f"{API_BASE}/chat/completions").mock( + return_value=httpx.Response(200, json=_CHAT_REPLY) + ) + + await _router({"model": "openai/gpt-4.1"}, {"mode": "responses"}).acompletion( + model="m", messages=[{"role": "user", "content": "hi"}], _skip_responses_api_bridge=True + ) + + assert chat.call_count == 1 diff --git a/tests/unit/native/test_route_plan.py b/tests/unit/native/test_route_plan.py index 4a4e6fabe0f..b56e45b8ee5 100644 --- a/tests/unit/native/test_route_plan.py +++ b/tests/unit/native/test_route_plan.py @@ -84,15 +84,11 @@ def test_responses_reach_chat_without_a_provider_config_or_when_asked( @pytest.mark.parametrize( - ("responses_mode_model", "skip_responses_api_bridge", "expected"), + ("has_config", "expected"), [ - (True, False, NativeResponses()), - (True, True, NativeChat()), - (False, False, NativeChat()), - (False, True, NativeChat()), + (True, NativeResponses()), + (False, NativeChat()), ], ) -def test_chat_bridges_to_responses_only_for_responses_mode_models( - responses_mode_model: bool, skip_responses_api_bridge: bool, expected: RoutePlan -): - assert plan_chat_route(responses_mode_model, skip_responses_api_bridge) == expected +def test_chat_bridges_to_responses_only_when_the_provider_serves_it(has_config: bool, expected: RoutePlan): + assert plan_chat_route(OpenAIResponsesAPIConfig() if has_config else None) == expected diff --git a/tests/unit/responses/litellm_completion_transformation/test_handler.py b/tests/unit/responses/litellm_completion_transformation/test_handler.py index d731bf5f41c..6897eadaa53 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_handler.py @@ -1,17 +1,3 @@ -"""Regression tests for the responses -> completion fallback bridge guard. - -When the Responses API falls back to chat completions (no native responses -config), it must tag the forwarded ``litellm.completion`` / ``litellm.acompletion`` -call with ``_skip_responses_api_bridge=True`` so ``completion()`` does not bridge -the request straight back to the Responses API and mutually recurse forever. - -Both fallback paths are covered: the sync ``response_api_handler`` (``_is_async`` -False) and the async ``async_response_api_handler`` (``_is_async`` True). The -module-level ``litellm.completion`` / ``litellm.acompletion`` are patched to -capture the forwarded kwargs; if the flag-setting line is removed the captured -kwargs lack the flag and these tests fail. -""" - import json from collections.abc import Mapping, Sequence from types import SimpleNamespace @@ -35,50 +21,6 @@ class _StopForwarding(Exception): """Raised by the mocked (a)completion once the forwarded kwargs are captured.""" -def test_sync_fallback_tags_skip_responses_api_bridge(): - handler = LiteLLMCompletionTransformationHandler() - captured: dict = {} - - def fake_completion(**kwargs): - captured.update(kwargs) - raise _StopForwarding() - - with patch("litellm.completion", fake_completion): - with pytest.raises(_StopForwarding): - handler.response_api_handler( - model="gpt-4o", - input="hello", - responses_api_request={}, - custom_llm_provider="openai", - _is_async=False, - ) - - assert captured.get("_skip_responses_api_bridge") is True - - -@pytest.mark.asyncio -async def test_async_fallback_tags_skip_responses_api_bridge(): - handler = LiteLLMCompletionTransformationHandler() - captured: dict = {} - - async def fake_acompletion(**kwargs): - captured.update(kwargs) - raise _StopForwarding() - - with patch("litellm.acompletion", fake_acompletion): - coro = handler.response_api_handler( - model="gpt-4o", - input="hello", - responses_api_request={}, - custom_llm_provider="openai", - _is_async=True, - ) - with pytest.raises(_StopForwarding): - await coro - - assert captured.get("_skip_responses_api_bridge") is True - - _CODEX_ADDITIONAL_TOOLS_ITEM = { "type": "additional_tools", "id": "at_codex", diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 68c031d9778..2da266b9912 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -49,7 +49,14 @@ from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.litellm_params import ControlOptions from litellm.types.llms.openai import AllMessageValues, HttpxBinaryResponseContent from litellm.types.prompts.init_prompts import PromptSpec -from litellm.types.utils import Delta, ModelResponseStream, StandardCallbackDynamicParams, StreamingChoices, Usage +from litellm.types.utils import ( + CallTypes, + Delta, + ModelResponseStream, + StandardCallbackDynamicParams, + StreamingChoices, + Usage, +) from tests.unit.llms.bedrock.slow_upstream import ( STREAM_TIMEOUT_SECONDS, UPSTREAM_FIRST_BYTE_SECONDS, @@ -446,6 +453,42 @@ def _settled_success_calls(counter: _SyncSuccessCounter, expected: int) -> int: return len(counter.calls) +class _DeploymentHookCounter(CustomLogger): + call_types: tuple[str, ...] = () + + async def async_pre_call_deployment_hook(self, kwargs: dict[str, object], call_type: CallTypes | None) -> None: + self.call_types = (*self.call_types, str(call_type)) + + +@pytest.mark.asyncio +async def test_responses_mode_model_without_a_responses_api_stays_on_chat( + respx_mock: respx.MockRouter, openai_api_response: dict[str, object], monkeypatch: pytest.MonkeyPatch +): + counter: Final = _DeploymentHookCounter() + monkeypatch.setattr(litellm, "callbacks", [counter]) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + litellm.register_model( + {"groq/responses-only-model": {"litellm_provider": "groq", "mode": "responses", "max_tokens": 100}} + ) + api_base: Final = "https://example.invalid/openai/v1" + chat_route: Final = respx_mock.post(f"{api_base}/chat/completions").mock( + return_value=httpx.Response(status_code=200, json=openai_api_response) + ) + + response: Final = await litellm.acompletion( + model="groq/responses-only-model", + api_base=api_base, + api_key="sk-x", + messages=[{"role": "user", "content": "hi"}], + ) + + assert isinstance(response, litellm.ModelResponse) + assert response.id == openai_api_response["id"] + assert chat_route.call_count == 1 + assert counter.call_types == ("CallTypes.acompletion",) + + def test_execute_chat_calls_the_provider_without_logging( respx_mock: respx.MockRouter, openai_api_response, monkeypatch: pytest.MonkeyPatch ): @@ -472,7 +515,6 @@ def test_execute_chat_calls_the_provider_without_logging( kwargs={"api_base": api_base, "api_key": "sk-x", "litellm_logging_obj": logging_obj}, non_default_params={}, control_options=ControlOptions(), - skip_responses_api_bridge=False, prompt_id=None, prompt_variables=None, )