diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 07d5e6314dd..e4c1c9fc5cf 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1816,16 +1816,19 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, Any]]: This helper uses ``json.JSONDecoder.raw_decode()`` to walk the string and extract each JSON object individually. + The walk degrades gracefully: if the string is malformed or truncated + (e.g. a stream that ended mid-tool-call), whatever complete objects were + parsed before the bad tail are returned and the remainder is discarded + with a warning, rather than raising. The sole caller + (``_convert_to_bedrock_tool_call_invoke``) treats an empty result as + ``input={}`` so the conversation can continue instead of hard-failing. + Returns ------- list[dict] A list of parsed dicts – one per JSON object found. If *raw* is - empty or whitespace-only, an empty list is returned. - - Raises - ------ - json.JSONDecodeError - If the string contains text that cannot be parsed as JSON at all. + empty, whitespace-only, or wholly unparseable, an empty list is + returned. """ import json @@ -1845,7 +1848,17 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, Any]]: if idx >= length: break - obj, end_idx = decoder.raw_decode(raw, idx) + try: + obj, end_idx = decoder.raw_decode(raw, idx) + except json.JSONDecodeError as e: + verbose_logger.warning( + "split_concatenated_json_objects: discarding unparseable tool-call " + "arguments tail after %d complete object(s); decode_start=%d error=%s", + len(results), + idx, + e, + ) + break if isinstance(obj, dict): results.append(obj) else: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 2ffe015c727..0ed15c43ccf 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -3712,7 +3712,13 @@ def _convert_to_bedrock_tool_call_invoke( _parts_list.append(cache_point_block) return _parts_list except Exception as e: - raise Exception(f"Unable to convert openai tool calls={tool_calls} to bedrock tool calls. Received error={e}") + tool_call_ids: Final = tuple(tool.get("id") for tool in tool_calls if isinstance(tool, dict)) + raise litellm.BadRequestError( + message=f"Unable to convert openai tool calls with ids={tool_call_ids} to bedrock tool calls. " + f"Received error={e}", + model=model or "", + llm_provider="bedrock", + ) from e def _append_bedrock_tool_result_media_block( diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index af40245ebfa..f9311497729 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -241,10 +241,32 @@ def test_split_concatenated_json_non_dict_value(): assert result == [{}] -def test_split_concatenated_json_invalid_raises(): - """Completely invalid JSON raises JSONDecodeError.""" - with pytest.raises(json.JSONDecodeError): - split_concatenated_json_objects("not json at all") +def test_split_concatenated_json_wholly_invalid_returns_empty(): + """ + Wholly unparseable JSON degrades to an empty list instead of raising. + + Regression for https://github.com/BerriAI/litellm/issues/18667: a raise + here propagated out of `_convert_to_bedrock_tool_call_invoke` and turned + every replayed conversation into a 500. + """ + assert split_concatenated_json_objects("not json at all") == [] + + +def test_split_concatenated_json_malformed_object_returns_empty(): + """ + A single malformed object (missing comma between keys) degrades to an + empty list rather than raising `Expecting ',' delimiter`. + """ + assert split_concatenated_json_objects('{"location": "Boston" "unit": "celsius"}') == [] + + +def test_split_concatenated_json_salvages_prefix_before_truncated_tail(): + """ + Complete objects parsed before an unparseable/truncated tail are kept; + only the bad tail is discarded. + """ + result = split_concatenated_json_objects('{"a": 1}{"b": 2}{"c":') + assert result == [{"a": 1}, {"b": 2}] # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index de5d0a180c6..fffbc884782 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -2287,6 +2287,116 @@ def test_bedrock_tool_call_invoke_non_dict_arguments(): assert result[0]["toolUse"]["input"] == {} +def test_bedrock_tool_call_invoke_malformed_json_does_not_raise(): + """ + Regression for https://github.com/BerriAI/litellm/issues/18667. + + When the model emits malformed JSON in tool-call arguments (here a + missing comma between keys), replaying that history must NOT raise + `Unable to convert openai tool calls ... Expecting ',' delimiter`. + It degrades to an empty-object input so the conversation can continue. + """ + tool_calls = [ + { + "id": "toolu_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "Boston" "unit": "celsius"}', + }, + } + ] + result = _convert_to_bedrock_tool_call_invoke(tool_calls) + assert len(result) == 1 + assert result[0]["toolUse"]["toolUseId"] == "toolu_abc123" + assert result[0]["toolUse"]["name"] == "get_weather" + assert result[0]["toolUse"]["input"] == {} + + +def test_bedrock_tool_call_invoke_salvages_valid_prefix_before_truncated_tail(): + """ + A valid leading object followed by a truncated tail keeps the valid + object rather than dropping everything or raising. + """ + tool_calls = [ + { + "id": "call_partial", + "type": "function", + "function": {"name": "shell", "arguments": '{"cmd": "ls"}{"cmd":'}, + } + ] + result = _convert_to_bedrock_tool_call_invoke(tool_calls) + assert len(result) == 1 + assert result[0]["toolUse"]["input"] == {"cmd": "ls"} + + +def test_bedrock_tool_call_invoke_mixed_turn_survives_one_malformed_call(): + """ + Regression for LIT-4574: an assistant turn with several tool calls where only one + has malformed/truncated arguments must keep the valid calls intact and degrade just + the bad one to empty input, instead of killing the entire turn. + """ + tool_calls = [ + { + "id": "t_good", + "type": "function", + "function": { + "name": "good_tool", + "arguments": '{"item_type": "email", "item_id": "AAMkAD=="}', + }, + }, + { + "id": "t_bad", + "type": "function", + "function": {"name": "bad_tool", "arguments": '{"item_type": "email"'}, + }, + ] + result = _convert_to_bedrock_tool_call_invoke(tool_calls) + tool_uses = [block["toolUse"] for block in result if "toolUse" in block] + assert len(tool_uses) == 2 + by_name = {tool_use["name"]: tool_use for tool_use in tool_uses} + assert by_name["good_tool"]["input"] == {"item_type": "email", "item_id": "AAMkAD=="} + assert by_name["bad_tool"]["input"] == {} + + +def test_bedrock_tool_call_invoke_truncated_json_arguments(): + """ + Truncated tool call arguments (issue #35303) must not raise. A client replaying a + partially streamed tool call would otherwise trigger a pre-network exception that the + router maps to a retryable APIConnectionError and retries through the fallback graph. + """ + tool_calls = [ + { + "id": "tooluse_MAh2QLVjBRkvi5QJkLQ08V", + "type": "function", + "function": { + "name": "replace_note_content", + "arguments": '{"note_id": "999af35c-4061-4ece-8581-7d43fc988ba4", "title": "WG"', + }, + } + ] + result = _convert_to_bedrock_tool_call_invoke(tool_calls) + assert len(result) == 1 + assert result[0]["toolUse"]["toolUseId"] == "tooluse_MAh2QLVjBRkvi5QJkLQ08V" + assert result[0]["toolUse"]["input"] == {} + + +def test_bedrock_tool_call_invoke_unconvertible_raises_non_retryable_bad_request(): + """ + Conversion failures are client input errors, so they must surface as a non-retryable + BadRequestError instead of a bare Exception that maps to APIConnectionError, and the + message must not embed the tool call payload (issue #35303). + """ + tool_calls = [{"id": "call_bad", "type": "function", "function": None}] + + with pytest.raises(litellm.BadRequestError) as exc_info: + _convert_to_bedrock_tool_call_invoke(tool_calls) + + assert exc_info.value.status_code == 400 + assert "call_bad" in str(exc_info.value) + assert "function" not in str(exc_info.value).split("Received error=")[0] + + def test_make_valid_bedrock_tool_name_preserves_hyphens(): assert make_valid_bedrock_tool_name("my-tool") == "my-tool" assert (