diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index a6f10e1ede3..87524d86c61 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -3,7 +3,7 @@ import json import re import time import traceback -from collections.abc import Iterable, Sequence +from collections.abc import Mapping, Sequence from typing import Final, Literal, cast import litellm @@ -151,6 +151,16 @@ def _clear_later_replay_slice_metadata(choice: StreamingChoices) -> None: del choice.enhancements +def _invalid_choices_message(response_object: Mapping[str, object]) -> str: + raw_keys: Final = list(response_object.keys()) + if "choices" not in response_object: + return f"LiteLLM: provider returned a response with no 'choices'. Raw keys: {raw_keys}" + return ( + f"LiteLLM: provider returned 'choices' that is not a list ({type(response_object['choices']).__name__}). " + f"Raw keys: {raw_keys}" + ) + + async def convert_to_streaming_response_async( response_object: dict | None = None, ): @@ -179,14 +189,12 @@ async def convert_to_streaming_response_async( choice_list: Final[list[StreamingChoices]] = [] - if not response_object.get("choices"): + if not isinstance(response_object.get("choices"), list): from litellm.exceptions import APIError raise APIError( status_code=500, - message=( - f"LiteLLM: provider returned a response with no 'choices'. Raw keys: {list(response_object.keys())}" - ), + message=_invalid_choices_message(response_object), llm_provider="", model="", ) @@ -287,14 +295,12 @@ def convert_to_streaming_response( model_response_object: Final = ModelResponseStream() choice_list: Final[list[StreamingChoices]] = [] - if not response_object.get("choices"): + if not isinstance(response_object.get("choices"), list): from litellm.exceptions import APIError raise APIError( status_code=500, - message=( - f"LiteLLM: provider returned a response with no 'choices'. Raw keys: {list(response_object.keys())}" - ), + message=_invalid_choices_message(response_object), llm_provider="", model="", ) @@ -623,15 +629,12 @@ def convert_to_model_response_object( return convert_to_streaming_response(response_object=response_object) choice_list: Final[list[Choices]] = [] - if not response_object.get("choices") or not isinstance(response_object["choices"], Iterable): + if not isinstance(response_object.get("choices"), list): from litellm.exceptions import APIError raise APIError( status_code=500, - message=( - "LiteLLM: provider returned a response with no 'choices'. " - f"Raw keys: {list(response_object.keys())}" - ), + message=_invalid_choices_message(response_object), llm_provider="", model="", ) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 6ae17bac6ff..db23929e0c3 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1473,17 +1473,14 @@ class CustomStreamWrapper: self.received_finish_reason = response_obj["finish_reason"] elif self.custom_llm_provider == "cached_response": cached_chunk: Final = cast(ModelResponseStream, chunk) - chunk_finish_reason: Final = cached_chunk.choices[0].finish_reason + cached_choice: Final = cached_chunk.choices[0] if cached_chunk.choices else None + chunk_finish_reason: Final = cached_choice.finish_reason if cached_choice is not None else None response_obj = { - "text": cached_chunk.choices[0].delta.content, + "text": cached_choice.delta.content if cached_choice is not None else None, "is_finished": chunk_finish_reason is not None, "finish_reason": chunk_finish_reason, "original_chunk": cached_chunk, - "tool_calls": ( - cached_chunk.choices[0].delta.tool_calls - if hasattr(cached_chunk.choices[0].delta, "tool_calls") - else None - ), + "tool_calls": (getattr(cached_choice.delta, "tool_calls", None) if cached_choice is not None else None), } completion_obj["content"] = response_obj["text"] diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index db890662132..4bce760e943 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1487,8 +1487,9 @@ class LiteLLMAnthropicMessagesAdapter: anthropic_content.insert(0, polyfill_result.compaction_block) ## extract finish reason + openai_finish_reason: Final = response.choices[0].finish_reason if response.choices else "stop" translated_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic( - openai_finish_reason=response.choices[0].finish_reason + openai_finish_reason=openai_finish_reason ) anthropic_finish_reason: Final = ( "refusal" diff --git a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py index b6e30ddc711..31c554985a7 100644 --- a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py +++ b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py @@ -1623,15 +1623,11 @@ class TestMissingChoicesGuard: assert "no 'choices'" in exc_info.value.message - def test_convert_to_model_response_object_empty_choices_raises_api_error(self): - """Empty choices list raises APIError, same as missing/null choices. + def test_convert_to_model_response_object_empty_choices_returns_empty_list(self): + """An empty choices list is a real provider answer, so it converts to choices=[] instead of raising. - Provider-specific repair (e.g. github_copilot synthesizing choices for - Anthropic-native responses) happens before this guard, in the provider - config; the core utility keeps treating empty choices as an error. + See: https://github.com/BerriAI/litellm/issues/40276 """ - from litellm.exceptions import APIError - response_object = { "id": "msg_123", "model": "some-model", @@ -1639,16 +1635,17 @@ class TestMissingChoicesGuard: "usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, } - with pytest.raises(APIError) as exc_info: - convert_to_model_response_object( - response_object=response_object, - model_response_object=ModelResponse(), - ) + result = convert_to_model_response_object( + response_object=response_object, + model_response_object=ModelResponse(), + ) - assert "no 'choices'" in exc_info.value.message + assert isinstance(result, ModelResponse) + assert result.choices == [] + assert result.usage.prompt_tokens == 10 def test_convert_to_model_response_object_null_choices_raises_api_error(self): - """choices=None raises APIError.""" + """choices=None raises APIError that names the type instead of claiming the key is missing.""" from litellm.exceptions import APIError response_object = { @@ -1664,7 +1661,7 @@ class TestMissingChoicesGuard: model_response_object=ModelResponse(), ) - assert "no 'choices'" in exc_info.value.message + assert "'choices' that is not a list (NoneType)" in exc_info.value.message def test_convert_to_streaming_response_no_choices_raises_api_error(self): """Missing choices in streaming cache-hit path raises APIError.""" diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index 304d732c518..8e46ae21de6 100644 --- a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -1,4 +1,6 @@ +from typing import Final +import pytest from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -99,3 +101,97 @@ def test_handle_invalid_parallel_tool_calls_skips_custom_tool_calls(): ) result = _handle_invalid_parallel_tool_calls([custom_tool_call, function_tool_call]) assert result == [custom_tool_call, function_tool_call] + + +def test_convert_empty_choices_response() -> None: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_streaming_response, + ) + + resp: Final = { + "id": "x", + "created": 1, + "model": "gemini-3.5-flash", + "object": "chat.completion", + "choices": [], + "usage": {"prompt_tokens": 10, "completion_tokens": 0, "total_tokens": 10}, + "vertex_ai_safety_results": ["blocked"], + } + result: Final = convert_to_model_response_object( + response_object=resp, + model_response_object=ModelResponse(), + response_type="completion", + ) + assert result.choices == [] + assert getattr(result, "vertex_ai_safety_results") == ["blocked"] + + sync_stream: Final = list(convert_to_streaming_response(response_object=resp)) + assert len(sync_stream) == 1 + assert sync_stream[0].choices == [] + + +@pytest.mark.asyncio +async def test_convert_empty_choices_response_async() -> None: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_streaming_response_async, + ) + + resp: Final = { + "id": "x", + "created": 1, + "model": "gemini-3.5-flash", + "object": "chat.completion", + "choices": [], + "usage": {"prompt_tokens": 10, "completion_tokens": 0, "total_tokens": 10}, + } + async_chunks: Final = [chunk async for chunk in convert_to_streaming_response_async(response_object=resp)] + assert len(async_chunks) == 1 + assert async_chunks[0].choices == [] + + +def test_convert_missing_choices_raises_api_error() -> None: + from litellm.exceptions import APIError + + resp: Final = { + "id": "x", + "created": 1, + "model": "gemini-3.5-flash", + "object": "chat.completion", + } + with pytest.raises(APIError) as exc_info: + convert_to_model_response_object( + response_object=resp, + model_response_object=ModelResponse(), + response_type="completion", + ) + assert "no 'choices'" in str(exc_info.value) + + +@pytest.mark.parametrize(("choices", "type_name"), [({}, "dict"), ("", "str"), (None, "NoneType"), (0, "int")]) +@pytest.mark.asyncio +async def test_convert_non_list_choices_raises_api_error(choices: object, type_name: str) -> None: + from litellm.exceptions import APIError + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_streaming_response, + convert_to_streaming_response_async, + ) + + resp: Final = { + "id": "x", + "created": 1, + "model": "gemini-3.5-flash", + "object": "chat.completion", + "choices": choices, + } + expected: Final = f"'choices' that is not a list \\({type_name}\\)" + with pytest.raises(APIError, match=expected): + convert_to_model_response_object( + response_object=resp, + model_response_object=ModelResponse(), + response_type="completion", + ) + with pytest.raises(APIError, match=expected): + list(convert_to_streaming_response(response_object=resp)) + with pytest.raises(APIError, match=expected): + async for _ in convert_to_streaming_response_async(response_object=resp): + pass diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 0aa73833677..37e2031fdf4 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -6,7 +6,7 @@ import pytest import asyncio import traceback -from typing import Optional +from typing import Final, Optional import litellm from litellm import verbose_logger @@ -2633,6 +2633,48 @@ def test_dispatch_cached_response_extracts_delta( assert initialized_custom_stream_wrapper.response_id == "chatcmpl-cache-1" +def test_dispatch_cached_response_without_choices_is_an_empty_chunk( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + """A cached completion with no choices replays as an empty, unfinished chunk + instead of raising IndexError on choices[0].""" + initialized_custom_stream_wrapper.custom_llm_provider = "cached_response" + chunk: Final = ModelResponseStream(id="chatcmpl-cache-empty", choices=[]) + + result, model_response, completion_obj = _run_dispatch( + initialized_custom_stream_wrapper, chunk + ) + + assert isinstance(result, _ProviderChunkParsed) + assert completion_obj["content"] is None + assert initialized_custom_stream_wrapper.received_finish_reason is None + assert model_response.id == "chatcmpl-cache-empty" + + +@pytest.mark.asyncio +async def test_cached_response_without_choices_streams_a_single_stop_chunk( + logging_obj: Logging, +): + """A stream cache hit on a completion stored with choices == [] ends with one + finish_reason=stop chunk, the same shape the live empty stream produced.""" + + async def cached_chunks(): + yield ModelResponseStream(id="chatcmpl-cache-empty", choices=[]) + + wrapper: Final = CustomStreamWrapper( + completion_stream=cached_chunks(), + model="test-model", + logging_obj=logging_obj, + custom_llm_provider="cached_response", + ) + + chunks: Final = tuple([chunk async for chunk in wrapper]) + + assert len(chunks) == 1 + assert tuple(choice.finish_reason for chunk in chunks for choice in chunk.choices) == ("stop",) + assert all(choice.delta.content in (None, "") for chunk in chunks for choice in chunk.choices) + + def test_dispatch_vertex_ai_legacy_text_and_finish_reason( initialized_custom_stream_wrapper: CustomStreamWrapper, ): diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index c59ec70b015..00b3a0633f2 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -41,6 +41,21 @@ from litellm.types.utils import ( ) +def test_translate_openai_response_to_anthropic_empty_choices() -> None: + response: Final = ModelResponse( + id="chatcmpl-empty", + model="gemini-3.5-flash", + choices=[], + usage=Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10), + ) + + result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) + + assert result["content"] == [] + assert result["stop_reason"] == "end_turn" + assert result["usage"]["input_tokens"] == 10 + + def test_translate_chat_refusal_to_anthropic_response(): response = ModelResponse( id="chatcmpl-refusal", diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 038df3656fe..de6322f27ee 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -120,7 +120,7 @@ def test_completion_missing_role(openai_api_response): print(f"openai_api_response: {openai_api_response}") with patch.object( - client.chat.completions.with_raw_response, "create", mock_raw_response + client.chat.completions.with_raw_response, "create", MagicMock(return_value=mock_raw_response) ) as mock_create: litellm.completion( model="gpt-4o-mini",