fix(anthropic): decode deeply nested JSON mode answers without slow recursive validation

litellm/llms/base_llm/base_utils.py

@@ -13,11 +13,11 @@ from abc import ABC, abstractmethod
 from collections.abc import Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, TypeAlias
+from typing import Any, Final, TypeAlias
 
 from openai.lib import _parsing, _pydantic  # pyright: ignore[reportPrivateUsage]  # SDK parser internals
-from pydantic import BaseModel, InstanceOf, JsonValue, TypeAdapter, ValidationError
-from typing_extensions import TypeAliasType, TypeIs  # noqa: TID251  # neither is in typing on Python 3.10
+from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
+from typing_extensions import TypeIs  # noqa: TID251  # narrows a message to its system member by the role literal
 
 from litellm._logging import verbose_logger
 from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX
@@ -211,22 +211,11 @@ class SourceNumber:
 
 
 _JsonScalar: TypeAlias = str | bool | int | float | None
-if TYPE_CHECKING:
-    AnswerValue: TypeAlias = (
-        InstanceOf[SourceNumber] | _JsonScalar | Sequence["AnswerValue"] | Mapping[str, "AnswerValue"]
-    )
-else:
-    AnswerValue = TypeAliasType(
-        "AnswerValue", InstanceOf[SourceNumber] | _JsonScalar | Sequence["AnswerValue"] | Mapping[str, "AnswerValue"]
-    )
-_ANSWER_OBJECT: Final = TypeAdapter(Mapping[str, AnswerValue])
+AnswerValue: TypeAlias = SourceNumber | _JsonScalar | Sequence["AnswerValue"] | Mapping[str, "AnswerValue"]
 
 
-def as_answer_object(value: object) -> Mapping[str, AnswerValue] | None:
-    try:
-        return _ANSWER_OBJECT.validate_python(value)
-    except ValidationError:
-        return None
+def as_answer_object(value: AnswerValue) -> Mapping[str, AnswerValue] | None:
+    return value if isinstance(value, Mapping) else None
 
 
 def _source_float(text: str) -> float | SourceNumber:
@@ -242,11 +231,18 @@ def _source_int(text: str) -> int | SourceNumber:
         return SourceNumber(text)
 
 
+def _is_decoded_json_object(
+    decoded: object,
+) -> TypeIs[Mapping[str, AnswerValue]]:  # guard-ok: json.loads with the source-number hooks yields only answer values
+    return isinstance(decoded, dict)
+
+
 def decoded_answer_object(text: str) -> Mapping[str, AnswerValue] | None:
     try:
-        return _ANSWER_OBJECT.validate_python(json.loads(text, parse_float=_source_float, parse_int=_source_int))
-    except (ValidationError, json.JSONDecodeError, RecursionError):
+        decoded: Final[object] = json.loads(text, parse_float=_source_float, parse_int=_source_int)
+    except (json.JSONDecodeError, RecursionError):
         return None
+    return decoded if _is_decoded_json_object(decoded) else None
 
 
 def _marked_source_number(marker: str, number: SourceNumber) -> str:

tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py

@@ -3262,6 +3262,11 @@ _DIGITS_PAST_THE_INT_LIMIT: Final = "1" * (sys.get_int_max_str_digits() + 1)
         (('{"n": 1e5}',), '{"n": 1e5}'),
         (('{"values": false}',), "false"),
         (('{"values": null}',), '{"values": null}'),
+        pytest.param(
+            ('{"values": ' + "[" * 300 + "]" * 300 + "}",),
+            "[" * 300 + "]" * 300,
+            id="values-nested-300-levels",
+        ),
         pytest.param(
             ('{"values": ' + "[" * 100000 + "]" * 100000 + "}",),
             '{"values": ' + "[" * 100000 + "]" * 100000 + "}",

tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py

@@ -8584,6 +8584,40 @@ def test_convert_tool_response_to_message_keeps_arguments_too_deep_to_decode(arg
     assert message.content == arguments
 
 
+_DEEPLY_NESTED: Final = "[" * 300 + "]" * 300
+
+
+@pytest.mark.parametrize(
+    "arguments, schema, expected",
+    [
+        pytest.param('{"values": ' + _DEEPLY_NESTED + "}", None, _DEEPLY_NESTED, id="values_answer"),
+        pytest.param(
+            '{"name": "json_tool_call", "parameters": {"tier": ' + _DEEPLY_NESTED + ', "summary": "x"}}',
+            _RISK_SCHEMA,
+            '{"tier": ' + _DEEPLY_NESTED + ', "summary": "x"}',
+            id="envelope",
+        ),
+    ],
+)
+def test_convert_tool_response_to_message_unwraps_a_deeply_nested_answer(
+    arguments: str, schema: dict[str, object] | None, expected: str
+):
+    message = AnthropicConfig.convert_tool_response_to_message(
+        tool_calls=[
+            ChatCompletionToolCallChunk(
+                id="toolu_1",
+                type="function",
+                function=ChatCompletionToolCallFunctionChunk(name=RESPONSE_FORMAT_TOOL_NAME, arguments=arguments),
+                index=0,
+            )
+        ],
+        response_format_schema=None if schema is None else _schema(schema),
+    )
+
+    assert message is not None
+    assert message.content == expected
+
+
 _DIGITS_PAST_THE_INT_LIMIT: Final = "1" * (sys.get_int_max_str_digits() + 1)