Merge pull request #33842 from BerriAI/litellm_fix_bedrock_malformed_toolcall_18667

fix(bedrock): degrade gracefully on malformed tool-call arguments

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -1816,16 +1816,19 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, Any]]:
     This helper uses ``json.JSONDecoder.raw_decode()`` to walk the string
     and extract each JSON object individually.
 
+    The walk degrades gracefully: if the string is malformed or truncated
+    (e.g. a stream that ended mid-tool-call), whatever complete objects were
+    parsed before the bad tail are returned and the remainder is discarded
+    with a warning, rather than raising.  The sole caller
+    (``_convert_to_bedrock_tool_call_invoke``) treats an empty result as
+    ``input={}`` so the conversation can continue instead of hard-failing.
+
     Returns
     -------
     list[dict]
         A list of parsed dicts – one per JSON object found.  If *raw* is
-        empty or whitespace-only, an empty list is returned.
-
-    Raises
-    ------
-    json.JSONDecodeError
-        If the string contains text that cannot be parsed as JSON at all.
+        empty, whitespace-only, or wholly unparseable, an empty list is
+        returned.
     """
     import json
 
@@ -1845,7 +1848,17 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, Any]]:
         if idx >= length:
             break
 
-        obj, end_idx = decoder.raw_decode(raw, idx)
+        try:
+            obj, end_idx = decoder.raw_decode(raw, idx)
+        except json.JSONDecodeError as e:
+            verbose_logger.warning(
+                "split_concatenated_json_objects: discarding unparseable tool-call "
+                "arguments tail after %d complete object(s); decode_start=%d error=%s",
+                len(results),
+                idx,
+                e,
+            )
+            break
         if isinstance(obj, dict):
             results.append(obj)
         else:

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -3712,7 +3712,13 @@ def _convert_to_bedrock_tool_call_invoke(
                         _parts_list.append(cache_point_block)
         return _parts_list
     except Exception as e:
-        raise Exception(f"Unable to convert openai tool calls={tool_calls} to bedrock tool calls. Received error={e}")
+        tool_call_ids: Final = tuple(tool.get("id") for tool in tool_calls if isinstance(tool, dict))
+        raise litellm.BadRequestError(
+            message=f"Unable to convert openai tool calls with ids={tool_call_ids} to bedrock tool calls. "
+            f"Received error={e}",
+            model=model or "",
+            llm_provider="bedrock",
+        ) from e
 
 
 def _append_bedrock_tool_result_media_block(

tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py

@@ -241,10 +241,32 @@ def test_split_concatenated_json_non_dict_value():
     assert result == [{}]
 
 
-def test_split_concatenated_json_invalid_raises():
-    """Completely invalid JSON raises JSONDecodeError."""
-    with pytest.raises(json.JSONDecodeError):
-        split_concatenated_json_objects("not json at all")
+def test_split_concatenated_json_wholly_invalid_returns_empty():
+    """
+    Wholly unparseable JSON degrades to an empty list instead of raising.
+
+    Regression for https://github.com/BerriAI/litellm/issues/18667: a raise
+    here propagated out of `_convert_to_bedrock_tool_call_invoke` and turned
+    every replayed conversation into a 500.
+    """
+    assert split_concatenated_json_objects("not json at all") == []
+
+
+def test_split_concatenated_json_malformed_object_returns_empty():
+    """
+    A single malformed object (missing comma between keys) degrades to an
+    empty list rather than raising `Expecting ',' delimiter`.
+    """
+    assert split_concatenated_json_objects('{"location": "Boston" "unit": "celsius"}') == []
+
+
+def test_split_concatenated_json_salvages_prefix_before_truncated_tail():
+    """
+    Complete objects parsed before an unparseable/truncated tail are kept;
+    only the bad tail is discarded.
+    """
+    result = split_concatenated_json_objects('{"a": 1}{"b": 2}{"c":')
+    assert result == [{"a": 1}, {"b": 2}]
 
 
 # ---------------------------------------------------------------------------

tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py

@@ -2287,6 +2287,116 @@ def test_bedrock_tool_call_invoke_non_dict_arguments():
     assert result[0]["toolUse"]["input"] == {}
 
 
+def test_bedrock_tool_call_invoke_malformed_json_does_not_raise():
+    """
+    Regression for https://github.com/BerriAI/litellm/issues/18667.
+
+    When the model emits malformed JSON in tool-call arguments (here a
+    missing comma between keys), replaying that history must NOT raise
+    `Unable to convert openai tool calls ... Expecting ',' delimiter`.
+    It degrades to an empty-object input so the conversation can continue.
+    """
+    tool_calls = [
+        {
+            "id": "toolu_abc123",
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "arguments": '{"location": "Boston" "unit": "celsius"}',
+            },
+        }
+    ]
+    result = _convert_to_bedrock_tool_call_invoke(tool_calls)
+    assert len(result) == 1
+    assert result[0]["toolUse"]["toolUseId"] == "toolu_abc123"
+    assert result[0]["toolUse"]["name"] == "get_weather"
+    assert result[0]["toolUse"]["input"] == {}
+
+
+def test_bedrock_tool_call_invoke_salvages_valid_prefix_before_truncated_tail():
+    """
+    A valid leading object followed by a truncated tail keeps the valid
+    object rather than dropping everything or raising.
+    """
+    tool_calls = [
+        {
+            "id": "call_partial",
+            "type": "function",
+            "function": {"name": "shell", "arguments": '{"cmd": "ls"}{"cmd":'},
+        }
+    ]
+    result = _convert_to_bedrock_tool_call_invoke(tool_calls)
+    assert len(result) == 1
+    assert result[0]["toolUse"]["input"] == {"cmd": "ls"}
+
+
+def test_bedrock_tool_call_invoke_mixed_turn_survives_one_malformed_call():
+    """
+    Regression for LIT-4574: an assistant turn with several tool calls where only one
+    has malformed/truncated arguments must keep the valid calls intact and degrade just
+    the bad one to empty input, instead of killing the entire turn.
+    """
+    tool_calls = [
+        {
+            "id": "t_good",
+            "type": "function",
+            "function": {
+                "name": "good_tool",
+                "arguments": '{"item_type": "email", "item_id": "AAMkAD=="}',
+            },
+        },
+        {
+            "id": "t_bad",
+            "type": "function",
+            "function": {"name": "bad_tool", "arguments": '{"item_type": "email"'},
+        },
+    ]
+    result = _convert_to_bedrock_tool_call_invoke(tool_calls)
+    tool_uses = [block["toolUse"] for block in result if "toolUse" in block]
+    assert len(tool_uses) == 2
+    by_name = {tool_use["name"]: tool_use for tool_use in tool_uses}
+    assert by_name["good_tool"]["input"] == {"item_type": "email", "item_id": "AAMkAD=="}
+    assert by_name["bad_tool"]["input"] == {}
+
+
+def test_bedrock_tool_call_invoke_truncated_json_arguments():
+    """
+    Truncated tool call arguments (issue #35303) must not raise. A client replaying a
+    partially streamed tool call would otherwise trigger a pre-network exception that the
+    router maps to a retryable APIConnectionError and retries through the fallback graph.
+    """
+    tool_calls = [
+        {
+            "id": "tooluse_MAh2QLVjBRkvi5QJkLQ08V",
+            "type": "function",
+            "function": {
+                "name": "replace_note_content",
+                "arguments": '{"note_id": "999af35c-4061-4ece-8581-7d43fc988ba4", "title": "WG"',
+            },
+        }
+    ]
+    result = _convert_to_bedrock_tool_call_invoke(tool_calls)
+    assert len(result) == 1
+    assert result[0]["toolUse"]["toolUseId"] == "tooluse_MAh2QLVjBRkvi5QJkLQ08V"
+    assert result[0]["toolUse"]["input"] == {}
+
+
+def test_bedrock_tool_call_invoke_unconvertible_raises_non_retryable_bad_request():
+    """
+    Conversion failures are client input errors, so they must surface as a non-retryable
+    BadRequestError instead of a bare Exception that maps to APIConnectionError, and the
+    message must not embed the tool call payload (issue #35303).
+    """
+    tool_calls = [{"id": "call_bad", "type": "function", "function": None}]
+
+    with pytest.raises(litellm.BadRequestError) as exc_info:
+        _convert_to_bedrock_tool_call_invoke(tool_calls)
+
+    assert exc_info.value.status_code == 400
+    assert "call_bad" in str(exc_info.value)
+    assert "function" not in str(exc_info.value).split("Received error=")[0]
+
+
 def test_make_valid_bedrock_tool_name_preserves_hyphens():
     assert make_valid_bedrock_tool_name("my-tool") == "my-tool"
     assert (