mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-14 04:09:09 +00:00
fix(vertex_ai): unwrap tool-emulated json_schema envelopes and send output_config.format for Claude (#45871)
* fix(vertex_ai): unwrap tool-emulated json_schema envelopes and send output_config.format for Claude Vertex AI Claude deployments that emulate response_format through a forced tool sometimes nest the schema object under an extra key such as `parameters` or `input`, and that envelope reached message.content with a 200. The converter now unwraps exactly one nested object that matches the schema, leaving a payload that is itself schema-shaped alone. Native structured-output rows sent the deprecated top-level `output_format`, which Vertex rejects with a 400. The Vertex request now carries it as `output_config.format`, with an explicit caller value winning. * fix(anthropic): unwrap tool-emulated json_schema envelopes on streams and tool-name wrappers * test(anthropic): expect the json tool description through a shared constant * fix(anthropic): drop the json tool description that makes sonnet-5 omit required keys * fix(anthropic): keep schema-accepted answers whole and decide held streams at the second key An answer the json_schema accepts is returned as is before any envelope rule runs, the tool-name wrapper is unwrapped only when the schema rejects it, a malformed schema turns the unwrap off, a streamed answer whose first key is a schema property named name or tool_name waits for its second key before streaming, the held fragments go through a StringIO, and a prefix that stays undecided past 4 KB is held to the end * test(anthropic): keep the stream family cells to providers whose stream iterator gets the schema * fix(anthropic): unwrap json tool envelopes by structure instead of schema validation The unwrap now reads only the schema's top-level property names. A lone undeclared key holding an object, alone or beside a name equal to the json tool's own name, is the envelope, and the unwrap repeats while the inner object shares no key with the schema. jsonschema and referencing are no longer imported on the request path. The stream holds only an answer that opens with the tool's name or with an undeclared key holding an object, and a schema without top-level properties streams as it arrives. * fix(anthropic): walk json tool envelopes iteratively * fix(anthropic): keep json tool arguments too deep to decode instead of raising * fix(anthropic): unwrap json tool envelopes beside undeclared string labels * fix(anthropic): keep json answers whole on schemas that allow extra keys and stream held text as sent A schema that allows undeclared keys and requires none now keeps an undeclared wrapper key on both the stream and non-stream paths. When no envelope is found, the held stream text is now sent exactly as the model wrote it. On Vertex /v1/messages, output_format now wins over output_config.format when both are set, which matches the Anthropic adapter. * test(anthropic): cover label envelopes, dynamic-key schemas, as-written held streams and output_format precedence --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
ccdca3d39c
commit
c581c85559
13 changed files with 3216 additions and 124 deletions
|
|
@ -3,7 +3,9 @@ Calling + translation logic for anthropic's `/v1/messages` endpoint
|
|||
"""
|
||||
|
||||
import copy
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Union, cast
|
||||
|
|
@ -20,6 +22,11 @@ from litellm.anthropic_beta_headers_manager import (
|
|||
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
|
||||
from litellm.llms.base_llm.base_utils import (
|
||||
RESPONSE_FORMAT_ENVELOPE_NAME_KEYS,
|
||||
ResponseFormatSchema,
|
||||
response_format_envelope_contents,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
|
|
@ -57,7 +64,7 @@ from litellm.types.utils import (
|
|||
|
||||
from ...base import BaseLLM
|
||||
from ..common_utils import AnthropicError, process_anthropic_headers
|
||||
from .transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY, AnthropicConfig
|
||||
from .transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY, AnthropicConfig, JsonModeRequest
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -69,6 +76,67 @@ def _loads_stream_chunk(payload: str) -> dict[str, object]:
|
|||
return json.loads(payload)
|
||||
|
||||
|
||||
_EnvelopeVerdict = Literal["undecided", "pass_through", "hold"]
|
||||
_FIRST_OBJECT_KEY: Final = re.compile(r'^\s*\{\s*"((?:[^"\\]|\\.)*)"\s*:')
|
||||
_OBJECT_KEY_PREFIX: Final = re.compile(r'^\s*(?:\{\s*(?:"(?:[^"\\]|\\.)*\\?(?:"\s*)?)?)?$')
|
||||
_ENVELOPE_NAME_KEY_PATTERN: Final = "|".join(sorted(RESPONSE_FORMAT_ENVELOPE_NAME_KEYS))
|
||||
_FIRST_NAME_VALUE: Final = re.compile(r'^\s*\{\s*"(?:' + _ENVELOPE_NAME_KEY_PATTERN + r')"\s*:\s*"((?:[^"\\]|\\.)*)"')
|
||||
_NAME_VALUE_PREFIX: Final = re.compile(
|
||||
r'^\s*\{\s*"(?:' + _ENVELOPE_NAME_KEY_PATTERN + r')"\s*:\s*(?:"(?:[^"\\]|\\.)*\\?)?$'
|
||||
)
|
||||
_STRING_VALUE_THEN_KEY: Final = re.compile(r'^"(?:[^"\\]|\\.)*"\s*,\s*"((?:[^"\\]|\\.)*)"\s*:')
|
||||
_UNDECIDED_PREFIX_LIMIT: Final = 4096
|
||||
|
||||
|
||||
def _key_after_a_string_verdict(string_value_start: str, schema: ResponseFormatSchema) -> _EnvelopeVerdict:
|
||||
next_key: Final = _STRING_VALUE_THEN_KEY.match(string_value_start)
|
||||
if next_key is None:
|
||||
return "undecided"
|
||||
return "pass_through" if next_key.group(1) in schema.property_names else "hold"
|
||||
|
||||
|
||||
def _first_value_verdict(after_first_key: str, schema: ResponseFormatSchema) -> _EnvelopeVerdict:
|
||||
value_start: Final = after_first_key.lstrip()
|
||||
if not value_start:
|
||||
return "undecided"
|
||||
if value_start.startswith("{"):
|
||||
return "hold"
|
||||
if value_start.startswith('"') and schema.required_names:
|
||||
return _key_after_a_string_verdict(value_start, schema)
|
||||
return "pass_through"
|
||||
|
||||
|
||||
def _prefix_verdict(buffered_json: str, schema: ResponseFormatSchema, tool_name: str) -> _EnvelopeVerdict:
|
||||
first_key: Final = _FIRST_OBJECT_KEY.match(buffered_json)
|
||||
if first_key is None:
|
||||
return "undecided" if _OBJECT_KEY_PREFIX.match(buffered_json) else "pass_through"
|
||||
if first_key.group(1) in RESPONSE_FORMAT_ENVELOPE_NAME_KEYS:
|
||||
first_value: Final = _FIRST_NAME_VALUE.match(buffered_json)
|
||||
if first_value is not None and first_value.group(1) == tool_name:
|
||||
return "hold"
|
||||
if first_value is None and _NAME_VALUE_PREFIX.match(buffered_json):
|
||||
return "undecided"
|
||||
if first_key.group(1) in schema.property_names or not schema.property_names or schema.accepts_a_lone_undeclared_key:
|
||||
return "pass_through"
|
||||
return _first_value_verdict(buffered_json[first_key.end() :], schema)
|
||||
|
||||
|
||||
def _held_answer(held: str, schema: ResponseFormatSchema) -> str:
|
||||
try:
|
||||
payload: Final[object] = json.loads(held)
|
||||
except (json.JSONDecodeError, RecursionError):
|
||||
return held
|
||||
contents: Final = response_format_envelope_contents(payload, schema, RESPONSE_FORMAT_TOOL_NAME)
|
||||
return held if contents is None else json.dumps(contents)
|
||||
|
||||
|
||||
def _envelope_verdict(buffered_json: str, schema: ResponseFormatSchema, tool_name: str) -> _EnvelopeVerdict:
|
||||
verdict: Final = _prefix_verdict(buffered_json, schema, tool_name)
|
||||
if verdict == "undecided" and len(buffered_json) > _UNDECIDED_PREFIX_LIMIT:
|
||||
return "hold"
|
||||
return verdict
|
||||
|
||||
|
||||
async def make_call(
|
||||
client: AsyncHTTPHandler | None,
|
||||
api_base: str,
|
||||
|
|
@ -81,6 +149,7 @@ async def make_call(
|
|||
json_mode: bool,
|
||||
speed: str | None = None,
|
||||
tool_name_reverse_map: dict[str, str] | None = None,
|
||||
response_format_schema: ResponseFormatSchema | None = None,
|
||||
) -> tuple["ModelResponseIterator", httpx.Headers]:
|
||||
if client is None:
|
||||
client = litellm.module_level_aclient
|
||||
|
|
@ -116,6 +185,7 @@ async def make_call(
|
|||
json_mode=json_mode,
|
||||
speed=speed,
|
||||
tool_name_reverse_map=tool_name_reverse_map,
|
||||
response_format_schema=response_format_schema,
|
||||
)
|
||||
|
||||
# LOGGING
|
||||
|
|
@ -141,6 +211,7 @@ def make_sync_call(
|
|||
json_mode: bool,
|
||||
speed: str | None = None,
|
||||
tool_name_reverse_map: dict[str, str] | None = None,
|
||||
response_format_schema: ResponseFormatSchema | None = None,
|
||||
) -> tuple["ModelResponseIterator", httpx.Headers]:
|
||||
if client is None:
|
||||
client = litellm.module_level_client # re-use a module level client
|
||||
|
|
@ -184,6 +255,7 @@ def make_sync_call(
|
|||
json_mode=json_mode,
|
||||
speed=speed,
|
||||
tool_name_reverse_map=tool_name_reverse_map,
|
||||
response_format_schema=response_format_schema,
|
||||
)
|
||||
|
||||
# LOGGING
|
||||
|
|
@ -241,6 +313,9 @@ class AnthropicChatCompletion(BaseLLM):
|
|||
tool_name_reverse_map=(
|
||||
litellm_params.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY) if isinstance(litellm_params, dict) else None
|
||||
),
|
||||
response_format_schema=(
|
||||
JsonModeRequest.model_validate(optional_params).tool_schema() if json_mode and optional_params else None
|
||||
),
|
||||
)
|
||||
streamwrapper: Final = CustomStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
|
|
@ -485,6 +560,9 @@ class AnthropicChatCompletion(BaseLLM):
|
|||
if isinstance(litellm_params, dict)
|
||||
else None
|
||||
),
|
||||
response_format_schema=(
|
||||
JsonModeRequest.model_validate(optional_params).tool_schema() if json_mode else None
|
||||
),
|
||||
)
|
||||
return CustomStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
|
|
@ -550,6 +628,7 @@ class ModelResponseIterator:
|
|||
json_mode: bool | None = False,
|
||||
speed: str | None = None,
|
||||
tool_name_reverse_map: dict[str, str] | None = None,
|
||||
response_format_schema: ResponseFormatSchema | None = None,
|
||||
):
|
||||
self.streaming_response = streaming_response
|
||||
self.response_iterator = self.streaming_response
|
||||
|
|
@ -573,6 +652,9 @@ class ModelResponseIterator:
|
|||
self.is_response_format_tool: bool = False
|
||||
# Track if we've converted any response_format tools (affects finish_reason)
|
||||
self.converted_response_format_tool: bool = False
|
||||
self.response_format_schema = response_format_schema
|
||||
self._response_format_buffer: io.StringIO = io.StringIO()
|
||||
self._response_format_verdict: _EnvelopeVerdict = "undecided"
|
||||
|
||||
# For handling partial JSON chunks from fragmentation
|
||||
# See: https://github.com/BerriAI/litellm/issues/17473
|
||||
|
|
@ -955,6 +1037,8 @@ class ModelResponseIterator:
|
|||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
self._current_server_tool_id = None
|
||||
if self.is_response_format_tool:
|
||||
text = self._flush_response_format_buffer()
|
||||
# Reset response_format tool tracking when block stops
|
||||
self.is_response_format_tool = False
|
||||
# Reset current content block type
|
||||
|
|
@ -1072,6 +1156,10 @@ class ModelResponseIterator:
|
|||
tool_name: Final = tool_use.get("function", {}).get("name", "")
|
||||
self.is_response_format_tool = tool_name == RESPONSE_FORMAT_TOOL_NAME
|
||||
|
||||
if self.is_response_format_tool and self.response_format_schema is not None:
|
||||
self.converted_response_format_tool = True
|
||||
return self._stream_response_format_fragment(tool_use, self.response_format_schema), None
|
||||
|
||||
# Convert tool to content if we're tracking a response_format tool
|
||||
if self.is_response_format_tool:
|
||||
message: Final = AnthropicConfig.convert_tool_response_to_message(tool_calls=[tool_use])
|
||||
|
|
@ -1083,6 +1171,31 @@ class ModelResponseIterator:
|
|||
|
||||
return text, tool_use
|
||||
|
||||
def _stream_response_format_fragment(
|
||||
self, tool_use: ChatCompletionToolCallChunk, schema: ResponseFormatSchema
|
||||
) -> str:
|
||||
if tool_use.get("id") is not None:
|
||||
self._response_format_buffer = io.StringIO()
|
||||
self._response_format_verdict = "undecided"
|
||||
return ""
|
||||
fragment: Final = tool_use.get("function", {}).get("arguments") or ""
|
||||
if self._response_format_verdict == "pass_through":
|
||||
return fragment
|
||||
self._response_format_buffer.write(fragment)
|
||||
if self._response_format_verdict == "hold":
|
||||
return ""
|
||||
buffered: Final = self._response_format_buffer.getvalue()
|
||||
self._response_format_verdict = _envelope_verdict(buffered, schema, RESPONSE_FORMAT_TOOL_NAME)
|
||||
return buffered if self._response_format_verdict == "pass_through" else ""
|
||||
|
||||
def _flush_response_format_buffer(self) -> str:
|
||||
if self.response_format_schema is None or self._response_format_verdict == "pass_through":
|
||||
return ""
|
||||
held: Final = self._response_format_buffer.getvalue()
|
||||
self._response_format_buffer = io.StringIO()
|
||||
self._response_format_verdict = "undecided"
|
||||
return _held_answer(held, self.response_format_schema) if held else ""
|
||||
|
||||
def _handle_message_delta(self, chunk: dict) -> tuple[str, Usage | None, dict[str, Any] | None]:
|
||||
"""
|
||||
Handle message_delta event for finish_reason, usage, and container.
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ValidationError
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -35,7 +35,13 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
|
|||
place_mid_conversation_system,
|
||||
split_leading_system_run,
|
||||
)
|
||||
from litellm.llms.base_llm.base_utils import type_to_response_format_param
|
||||
from litellm.llms.base_llm.base_utils import (
|
||||
ResponseFormatSchema,
|
||||
as_json_object,
|
||||
response_format_schema,
|
||||
type_to_response_format_param,
|
||||
unwrap_response_format_envelope,
|
||||
)
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_ADVISOR_TOOL_TYPE,
|
||||
|
|
@ -1746,7 +1752,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
else:
|
||||
_input_schema.update(cast(AnthropicInputSchema, json_schema))
|
||||
|
||||
_tool: Final = AnthropicMessagesTool(name=RESPONSE_FORMAT_TOOL_NAME, input_schema=_input_schema)
|
||||
_tool: Final = AnthropicMessagesTool(
|
||||
name=RESPONSE_FORMAT_TOOL_NAME,
|
||||
input_schema=_input_schema,
|
||||
)
|
||||
return _tool
|
||||
|
||||
def should_strip_billing_metadata(self) -> bool:
|
||||
|
|
@ -2199,6 +2208,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
self,
|
||||
json_mode: bool | None,
|
||||
tool_calls: list[ChatCompletionToolCallChunk],
|
||||
response_format_schema: ResponseFormatSchema | None = None,
|
||||
) -> tuple[
|
||||
LitellmMessage | None,
|
||||
list[ChatCompletionToolCallChunk],
|
||||
|
|
@ -2218,11 +2228,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
json_tool: Final = tool_calls[json_indices[0]]
|
||||
if json_tool.get("function", {}).get("arguments") is None:
|
||||
return None, tool_calls, None
|
||||
_message: Final = AnthropicConfig.convert_tool_response_to_message(tool_calls=[json_tool])
|
||||
_message: Final = AnthropicConfig.convert_tool_response_to_message(
|
||||
tool_calls=[json_tool], response_format_schema=response_format_schema
|
||||
)
|
||||
return _message, [], None
|
||||
|
||||
first_json: Final = tool_calls[json_indices[0]]
|
||||
json_msg: Final = AnthropicConfig.convert_tool_response_to_message([first_json])
|
||||
json_msg: Final = AnthropicConfig.convert_tool_response_to_message(
|
||||
[first_json], response_format_schema=response_format_schema
|
||||
)
|
||||
extra_content: Final[str | None] = json_msg.content if json_msg is not None else None
|
||||
filtered_tools: Final = [t for i, t in enumerate(tool_calls) if i not in json_indices]
|
||||
return None, filtered_tools, extra_content
|
||||
|
|
@ -2658,6 +2672,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
prefix_prompt: str | None = None,
|
||||
speed: str | None = None,
|
||||
tool_name_reverse_map: dict[str, str] | None = None,
|
||||
response_format_schema: ResponseFormatSchema | None = None,
|
||||
):
|
||||
_hidden_params: Final[dict] = {}
|
||||
_hidden_params["additional_headers"] = process_anthropic_headers(dict(raw_response.headers))
|
||||
|
|
@ -2715,6 +2730,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
json_mode_message, tool_calls_for_message, json_extra_content = self._resolve_json_mode_non_streaming(
|
||||
json_mode=json_mode,
|
||||
tool_calls=tool_calls,
|
||||
response_format_schema=response_format_schema,
|
||||
)
|
||||
merged_text = text_content or ""
|
||||
if json_extra_content:
|
||||
|
|
@ -2823,12 +2839,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
prefix_prompt=prefix_prompt,
|
||||
speed=speed,
|
||||
tool_name_reverse_map=tool_name_reverse_map,
|
||||
response_format_schema=JsonModeRequest.model_validate(optional_params).tool_schema() if json_mode else None,
|
||||
)
|
||||
return model_response
|
||||
|
||||
@staticmethod
|
||||
def _convert_tool_response_to_message(
|
||||
tool_calls: list[ChatCompletionToolCallChunk],
|
||||
response_format_schema: ResponseFormatSchema | None = None,
|
||||
) -> LitellmMessage | None:
|
||||
"""
|
||||
In JSON mode, Anthropic API returns JSON schema as a tool call, we need to convert it to a message to follow the OpenAI format
|
||||
|
|
@ -2836,28 +2854,32 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
"""
|
||||
## HANDLE JSON MODE - anthropic returns single function call
|
||||
json_mode_content_str: Final[str | None] = tool_calls[0]["function"].get("arguments")
|
||||
if json_mode_content_str is None:
|
||||
return None
|
||||
try:
|
||||
if json_mode_content_str is not None:
|
||||
args: Final = json.loads(json_mode_content_str)
|
||||
if isinstance(args, dict) and (values := args.get("values")) is not None:
|
||||
_message = litellm.Message(content=json.dumps(values))
|
||||
return _message
|
||||
else:
|
||||
# a lot of the times the `values` key is not present in the tool response
|
||||
# relevant issue: https://github.com/BerriAI/litellm/issues/6741
|
||||
_message = litellm.Message(content=json.dumps(args))
|
||||
return _message
|
||||
except json.JSONDecodeError:
|
||||
payload: Final[object] = json.loads(json_mode_content_str)
|
||||
except (json.JSONDecodeError, RecursionError):
|
||||
# json decode error does occur, return the original tool response str
|
||||
return litellm.Message(content=json_mode_content_str)
|
||||
return None
|
||||
if response_format_schema is not None:
|
||||
return litellm.Message(
|
||||
content=json.dumps(
|
||||
unwrap_response_format_envelope(payload, response_format_schema, RESPONSE_FORMAT_TOOL_NAME)
|
||||
)
|
||||
)
|
||||
# a lot of the times the `values` key is not present in the tool response
|
||||
# relevant issue: https://github.com/BerriAI/litellm/issues/6741
|
||||
payload_object: Final = as_json_object(payload)
|
||||
values: Final = None if payload_object is None else payload_object.get("values")
|
||||
return litellm.Message(content=json.dumps(payload if values is None else values))
|
||||
|
||||
@classmethod
|
||||
def convert_tool_response_to_message(
|
||||
cls,
|
||||
tool_calls: list[ChatCompletionToolCallChunk], # mutable-ok: mirrors override contract
|
||||
response_format_schema: ResponseFormatSchema | None = None,
|
||||
) -> LitellmMessage | None:
|
||||
return cls._convert_tool_response_to_message(tool_calls)
|
||||
return cls._convert_tool_response_to_message(tool_calls, response_format_schema)
|
||||
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
|
||||
return AnthropicError(
|
||||
|
|
@ -2867,6 +2889,23 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
)
|
||||
|
||||
|
||||
_REQUEST_TOOLS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
|
||||
|
||||
class JsonModeRequest(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
tools: object = None
|
||||
|
||||
def tool_schema(self) -> ResponseFormatSchema | None:
|
||||
try:
|
||||
tools: Final = _REQUEST_TOOLS.validate_python(self.tools)
|
||||
except ValidationError:
|
||||
return None
|
||||
json_tool: Final = next((tool for tool in tools if tool.get("name") == RESPONSE_FORMAT_TOOL_NAME), None)
|
||||
return None if json_tool is None else response_format_schema(json_tool.get("input_schema"))
|
||||
|
||||
|
||||
def _valid_user_id(user_id: str) -> bool:
|
||||
"""
|
||||
Validate that user_id is not an email or phone number.
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import httpx
|
|||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion
|
||||
from litellm.llms.anthropic.chat.transformation import JsonModeRequest
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
|
|
@ -162,6 +163,9 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion):
|
|||
logging_obj=logging_obj,
|
||||
timeout=timeout,
|
||||
json_mode=json_mode,
|
||||
response_format_schema=(
|
||||
JsonModeRequest.model_validate(optional_params).tool_schema() if json_mode else None
|
||||
),
|
||||
)
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
process_anthropic_headers,
|
||||
|
|
|
|||
|
|
@ -3,13 +3,15 @@ Utility functions for base LLM classes.
|
|||
"""
|
||||
|
||||
import copy
|
||||
import itertools
|
||||
import json
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Final
|
||||
|
||||
from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk
|
||||
|
|
@ -157,6 +159,136 @@ def convert_tool_response_to_message(
|
|||
_convert_tool_response_to_message = convert_tool_response_to_message
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ResponseFormatSchema:
|
||||
property_names: frozenset[str]
|
||||
required_names: frozenset[str] = frozenset()
|
||||
allows_undeclared_keys: bool = False
|
||||
|
||||
@property
|
||||
def accepts_a_lone_undeclared_key(self) -> bool:
|
||||
return self.allows_undeclared_keys and not self.required_names
|
||||
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, JsonValue])
|
||||
_NAMES: Final = TypeAdapter(frozenset[str])
|
||||
|
||||
|
||||
def as_json_object(value: object) -> Mapping[str, JsonValue] | None:
|
||||
try:
|
||||
return _JSON_OBJECT.validate_python(value)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _names(value: object) -> frozenset[str]:
|
||||
try:
|
||||
return _NAMES.validate_python(value)
|
||||
except ValidationError:
|
||||
return frozenset()
|
||||
|
||||
|
||||
def _allows_undeclared_keys(document: Mapping[str, JsonValue]) -> bool:
|
||||
additional_properties: Final = document.get("additionalProperties")
|
||||
return (
|
||||
additional_properties is True
|
||||
or as_json_object(additional_properties) is not None
|
||||
or bool(as_json_object(document.get("patternProperties")))
|
||||
)
|
||||
|
||||
|
||||
def response_format_schema(value: object) -> ResponseFormatSchema | None:
|
||||
document: Final = as_json_object(value)
|
||||
if document is None:
|
||||
return None
|
||||
properties: Final = as_json_object(document.get("properties", {}))
|
||||
if properties is None:
|
||||
return None
|
||||
return ResponseFormatSchema(
|
||||
property_names=frozenset(properties),
|
||||
required_names=_names(document.get("required", ())),
|
||||
allows_undeclared_keys=_allows_undeclared_keys(document),
|
||||
)
|
||||
|
||||
|
||||
RESPONSE_FORMAT_ENVELOPE_NAME_KEYS: Final = frozenset({"name", "tool_name"})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _EnvelopeLayer:
|
||||
inner: Mapping[str, JsonValue]
|
||||
names_the_tool: bool
|
||||
|
||||
|
||||
def _json_object_or_encoded_object(value: JsonValue) -> Mapping[str, JsonValue] | None:
|
||||
if not isinstance(value, str):
|
||||
return as_json_object(value)
|
||||
try:
|
||||
return as_json_object(json.loads(value))
|
||||
except (json.JSONDecodeError, RecursionError):
|
||||
return None
|
||||
|
||||
|
||||
def _label_keys(payload_object: Mapping[str, JsonValue], schema: ResponseFormatSchema) -> frozenset[str]:
|
||||
if not schema.required_names:
|
||||
return frozenset()
|
||||
declared: Final = schema.property_names | schema.required_names
|
||||
return frozenset(key for key, value in payload_object.items() if isinstance(value, str) and key not in declared)
|
||||
|
||||
|
||||
def _envelope_layer(
|
||||
payload_object: Mapping[str, JsonValue], schema: ResponseFormatSchema, tool_name: str
|
||||
) -> _EnvelopeLayer | None:
|
||||
name_keys: Final = frozenset(
|
||||
key for key in RESPONSE_FORMAT_ENVELOPE_NAME_KEYS if payload_object.get(key) == tool_name
|
||||
)
|
||||
label_keys: Final = name_keys or _label_keys(payload_object, schema)
|
||||
wrapper_keys: Final = tuple(key for key in payload_object if key not in label_keys)
|
||||
if len(wrapper_keys) != 1 or wrapper_keys[0] in schema.property_names:
|
||||
return None
|
||||
wrapped: Final = payload_object[wrapper_keys[0]]
|
||||
inner: Final = _json_object_or_encoded_object(wrapped) if name_keys else as_json_object(wrapped)
|
||||
return None if inner is None else _EnvelopeLayer(inner=inner, names_the_tool=bool(name_keys))
|
||||
|
||||
|
||||
def _envelope_layers(
|
||||
payload_object: Mapping[str, JsonValue], schema: ResponseFormatSchema, tool_name: str
|
||||
) -> Iterator[_EnvelopeLayer]:
|
||||
layer = _envelope_layer(payload_object, schema, tool_name) # rebind-ok: the recursion gate needs a loop
|
||||
while layer is not None:
|
||||
yield layer
|
||||
layer = _envelope_layer(layer.inner, schema, tool_name)
|
||||
|
||||
|
||||
def _schema_object_inside_envelope(
|
||||
payload_object: Mapping[str, JsonValue], schema: ResponseFormatSchema, tool_name: str
|
||||
) -> Mapping[str, JsonValue] | None:
|
||||
layers: Final = _envelope_layers(payload_object, schema, tool_name)
|
||||
candidates: Final = layers if schema.property_names else itertools.islice(layers, 1)
|
||||
return next((layer.inner for layer in candidates if _holds_the_schema_object(layer, schema)), None)
|
||||
|
||||
|
||||
def _holds_the_schema_object(layer: _EnvelopeLayer, schema: ResponseFormatSchema) -> bool:
|
||||
if layer.names_the_tool:
|
||||
return True
|
||||
return not schema.accepts_a_lone_undeclared_key and not schema.property_names.isdisjoint(layer.inner)
|
||||
|
||||
|
||||
def response_format_envelope_contents(payload: object, schema: ResponseFormatSchema, tool_name: str) -> object | None:
|
||||
payload_object: Final = as_json_object(payload)
|
||||
if payload_object is None:
|
||||
return None
|
||||
schema_object: Final = _schema_object_inside_envelope(payload_object, schema, tool_name)
|
||||
if schema_object is not None:
|
||||
return schema_object
|
||||
return None if "values" in schema.property_names else payload_object.get("values")
|
||||
|
||||
|
||||
def unwrap_response_format_envelope(payload: object, schema: ResponseFormatSchema, tool_name: str) -> object:
|
||||
contents: Final = response_format_envelope_contents(payload, schema, tool_name)
|
||||
return payload if contents is None else contents
|
||||
|
||||
|
||||
def _dict_to_response_format_helper(response_format: dict, ref_template: str | None = None) -> dict:
|
||||
if ref_template is not None and response_format.get("type") == "json_schema":
|
||||
# Deep copy to avoid modifying original
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import json
|
||||
import types
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Final, cast
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
|
|
@ -16,6 +17,8 @@ from litellm.litellm_core_utils.optional_imports import ensure_optional_import
|
|||
from litellm.llms.anthropic.chat.handler import (
|
||||
ModelResponseIterator as AnthropicModelResponseIterator,
|
||||
)
|
||||
from litellm.llms.anthropic.chat.transformation import JsonModeRequest
|
||||
from litellm.llms.base_llm.base_utils import ResponseFormatSchema
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
|
|
@ -174,15 +177,34 @@ class AmazonCohereChatConfig:
|
|||
return optional_params
|
||||
|
||||
|
||||
_REQUEST_BODY: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def response_format_schema_from_request(data: str, json_mode: bool | None) -> ResponseFormatSchema | None:
|
||||
if not json_mode:
|
||||
return None
|
||||
try:
|
||||
body: Final = _REQUEST_BODY.validate_python(json.loads(data))
|
||||
except (json.JSONDecodeError, ValidationError):
|
||||
return None
|
||||
return JsonModeRequest.model_validate(body).tool_schema()
|
||||
|
||||
|
||||
def _stream_decoder(
|
||||
bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None,
|
||||
*,
|
||||
model: str,
|
||||
json_mode: bool | None,
|
||||
sync_stream: bool,
|
||||
response_format_schema: ResponseFormatSchema | None = None,
|
||||
) -> "AWSEventStreamDecoder":
|
||||
if bedrock_invoke_provider == "anthropic":
|
||||
return AmazonAnthropicClaudeStreamDecoder(model=model, sync_stream=sync_stream, json_mode=json_mode)
|
||||
return AmazonAnthropicClaudeStreamDecoder(
|
||||
model=model,
|
||||
sync_stream=sync_stream,
|
||||
json_mode=json_mode,
|
||||
response_format_schema=response_format_schema,
|
||||
)
|
||||
if bedrock_invoke_provider == "deepseek_r1":
|
||||
return AmazonDeepSeekR1StreamDecoder(model=model, sync_stream=sync_stream)
|
||||
if bedrock_invoke_provider == "moonshot":
|
||||
|
|
@ -250,7 +272,11 @@ async def make_call(
|
|||
)
|
||||
else:
|
||||
decoder: Final = _stream_decoder(
|
||||
bedrock_invoke_provider, model=model, json_mode=json_mode, sync_stream=False
|
||||
bedrock_invoke_provider,
|
||||
model=model,
|
||||
json_mode=json_mode,
|
||||
sync_stream=False,
|
||||
response_format_schema=response_format_schema_from_request(data, json_mode),
|
||||
)
|
||||
completion_stream = decoder.aiter_bytes(
|
||||
response.aiter_bytes(chunk_size=stream_chunk_size), response_headers=response.headers
|
||||
|
|
@ -341,7 +367,11 @@ def make_sync_call(
|
|||
)
|
||||
else:
|
||||
decoder: Final = _stream_decoder(
|
||||
bedrock_invoke_provider, model=model, json_mode=json_mode, sync_stream=True
|
||||
bedrock_invoke_provider,
|
||||
model=model,
|
||||
json_mode=json_mode,
|
||||
sync_stream=True,
|
||||
response_format_schema=response_format_schema_from_request(data, json_mode),
|
||||
)
|
||||
completion_stream = decoder.iter_bytes(
|
||||
response.iter_bytes(chunk_size=stream_chunk_size), response_headers=response.headers
|
||||
|
|
@ -878,6 +908,7 @@ class AmazonAnthropicClaudeStreamDecoder(AWSEventStreamDecoder):
|
|||
model: str,
|
||||
sync_stream: bool,
|
||||
json_mode: bool | None = None,
|
||||
response_format_schema: ResponseFormatSchema | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Child class of AWSEventStreamDecoder that handles the streaming response from the Anthropic family of models
|
||||
|
|
@ -889,6 +920,7 @@ class AmazonAnthropicClaudeStreamDecoder(AWSEventStreamDecoder):
|
|||
streaming_response=None,
|
||||
sync_stream=sync_stream,
|
||||
json_mode=json_mode,
|
||||
response_format_schema=response_format_schema,
|
||||
)
|
||||
|
||||
def _chunk_parser(self, chunk_data: dict) -> ModelResponseStream:
|
||||
|
|
|
|||
|
|
@ -10,8 +10,11 @@ import; extracting the helper into a leaf module resolves the warning and
|
|||
keeps the parent module's import surface narrow.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
# Keys inside ``output_config`` that Vertex AI Claude rejects regardless of
|
||||
# the target model. Add an entry only when a 400 "Extra inputs are not
|
||||
# permitted" is reproducible against the live Vertex endpoint for every model.
|
||||
|
|
@ -31,6 +34,24 @@ def _model_accepts_output_config_effort(model: str) -> bool:
|
|||
return AnthropicConfig.model_supports_effort_param(model, "vertex_ai")
|
||||
|
||||
|
||||
_OUTPUT_CONFIG: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _as_output_config(value: object) -> Mapping[str, object] | None:
|
||||
try:
|
||||
return _OUTPUT_CONFIG.validate_python(value)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _fold_output_format_into_output_config(
|
||||
output_config: Mapping[str, object] | None, output_format: object
|
||||
) -> Mapping[str, object] | None:
|
||||
if output_format is None:
|
||||
return output_config
|
||||
return {**(output_config or {}), "format": output_format}
|
||||
|
||||
|
||||
def sanitize_vertex_anthropic_output_params(data: dict, model: str) -> None:
|
||||
"""
|
||||
Strip Vertex-unsupported keys from ``output_config`` /
|
||||
|
|
@ -44,15 +65,18 @@ def sanitize_vertex_anthropic_output_params(data: dict, model: str) -> None:
|
|||
* Keys in ``VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS`` are always filtered.
|
||||
* ``output_config`` left empty after filtering is removed so the request
|
||||
body has no empty dict.
|
||||
* ``output_format`` is forwarded as-is (Vertex AI Claude accepts it).
|
||||
* ``output_format`` is moved into ``output_config.format``, because
|
||||
Vertex AI Claude rejects the top-level field as deprecated. When both
|
||||
are set, ``output_format`` wins, as it does in the Anthropic adapter.
|
||||
* Non-dict values for ``output_config`` are dropped to avoid sending
|
||||
malformed payloads downstream.
|
||||
"""
|
||||
output_config: Final = data.get("output_config")
|
||||
request: Final[dict[str, object]] = data # mutable-ok: the caller's request body is sanitized in place
|
||||
output_config: Final = _fold_output_format_into_output_config(
|
||||
_as_output_config(request.get("output_config")), request.pop("output_format", None)
|
||||
)
|
||||
if output_config is None:
|
||||
return
|
||||
if not isinstance(output_config, dict):
|
||||
data.pop("output_config", None)
|
||||
request.pop("output_config", None)
|
||||
return
|
||||
|
||||
drop_keys: Final = set(VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS)
|
||||
|
|
@ -68,6 +92,6 @@ def sanitize_vertex_anthropic_output_params(data: dict, model: str) -> None:
|
|||
|
||||
sanitized: Final = {k: v for k, v in output_config.items() if k not in drop_keys}
|
||||
if sanitized:
|
||||
data["output_config"] = sanitized
|
||||
request["output_config"] = sanitized
|
||||
else:
|
||||
data.pop("output_config", None)
|
||||
request.pop("output_config", None)
|
||||
|
|
|
|||
|
|
@ -159,7 +159,7 @@ class VertexAIAnthropicConfig(AnthropicConfig):
|
|||
"""
|
||||
Override parent method so VertexAI uses tool-based structured outputs
|
||||
unless the vertex map entry advertises native structured output
|
||||
(``output_format``, which Vertex AI Claude forwards for those models).
|
||||
(sent to Vertex AI Claude as ``output_config.format``).
|
||||
"""
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -13,7 +13,8 @@ import litellm
|
|||
from litellm._uuid import uuid
|
||||
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
|
||||
from litellm.llms.anthropic.chat.handler import ModelResponseIterator, make_call
|
||||
from litellm.llms.anthropic.chat.handler import ModelResponseIterator, _envelope_verdict, make_call
|
||||
from litellm.llms.base_llm.base_utils import response_format_schema, unwrap_response_format_envelope
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.types.llms.openai import (
|
||||
ChatCompletionToolCallChunk,
|
||||
|
|
@ -315,9 +316,7 @@ def test_streaming_thinking_blocks_are_replayable_after_signature_delta():
|
|||
assert reasoning_content == "Step 1. Step 2."
|
||||
assert thinking_blocks == (*expected_delta_blocks, expected_signature_block)
|
||||
assert "".join(block.get("thinking") or "" for block in thinking_blocks) == reasoning_content
|
||||
assert parsed_chunks[1].choices[0].delta.provider_specific_fields == {
|
||||
"thinking_blocks": [expected_delta_blocks[0]]
|
||||
}
|
||||
assert parsed_chunks[1].choices[0].delta.provider_specific_fields == {"thinking_blocks": [expected_delta_blocks[0]]}
|
||||
assert parsed_chunks[-1].choices[0].delta.provider_specific_fields == {
|
||||
"thinking_blocks": [expected_signature_block]
|
||||
}
|
||||
|
|
@ -356,7 +355,11 @@ def test_streamed_signed_thinking_round_trips_to_the_next_turn_once():
|
|||
"delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 1},
|
||||
{"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": None}, "usage": {"output_tokens": 30}},
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "tool_use", "stop_sequence": None},
|
||||
"usage": {"output_tokens": 30},
|
||||
},
|
||||
{"type": "message_stop"},
|
||||
)
|
||||
user_message: Final = {"role": "user", "content": "What's the weather in Paris?"}
|
||||
|
|
@ -843,13 +846,9 @@ async def test_anthropic_stream_options_return_usage_after_the_finish_chunk(
|
|||
},
|
||||
{"type": "message_stop"},
|
||||
)
|
||||
body: Final = "".join(
|
||||
f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events
|
||||
)
|
||||
body: Final = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events)
|
||||
route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=httpx.Response(
|
||||
200, text=body, headers={"content-type": "text/event-stream"}
|
||||
)
|
||||
return_value=httpx.Response(200, text=body, headers={"content-type": "text/event-stream"})
|
||||
)
|
||||
chunks: Final = await _collect_anthropic_stream(
|
||||
sync_mode,
|
||||
|
|
@ -860,9 +859,7 @@ async def test_anthropic_stream_options_return_usage_after_the_finish_chunk(
|
|||
stream_options={"include_usage": True},
|
||||
)
|
||||
finish_indices: Final = tuple(
|
||||
index
|
||||
for index, chunk in enumerate(chunks)
|
||||
if chunk.choices and chunk.choices[0].finish_reason is not None
|
||||
index for index, chunk in enumerate(chunks) if chunk.choices and chunk.choices[0].finish_reason is not None
|
||||
)
|
||||
finish_reasons: Final = tuple(
|
||||
chunk.choices[0].finish_reason
|
||||
|
|
@ -870,15 +867,11 @@ async def test_anthropic_stream_options_return_usage_after_the_finish_chunk(
|
|||
if chunk.choices and chunk.choices[0].finish_reason is not None
|
||||
)
|
||||
usage_indices: Final = tuple(
|
||||
index
|
||||
for index, chunk in enumerate(chunks)
|
||||
if getattr(chunk, "usage", None) is not None
|
||||
index for index, chunk in enumerate(chunks) if getattr(chunk, "usage", None) is not None
|
||||
)
|
||||
usage_chunk: Final = chunks[usage_indices[0]]
|
||||
|
||||
assert "".join(
|
||||
chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices
|
||||
) == "Blue sky."
|
||||
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) == "Blue sky."
|
||||
assert finish_reasons == ("stop",)
|
||||
assert finish_indices == (len(chunks) - 2,)
|
||||
assert usage_indices == (len(chunks) - 1,)
|
||||
|
|
@ -1460,17 +1453,11 @@ def test_accumulated_json_partial_fragment_returns_none_without_parsing():
|
|||
unlike Vertex which already deferred parsing until the buffer could close.
|
||||
A fragment that can't close a JSON value must not trigger a decode attempt.
|
||||
"""
|
||||
iterator = ModelResponseIterator(
|
||||
streaming_response=MagicMock(), sync_stream=True, json_mode=False
|
||||
)
|
||||
iterator = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=True, json_mode=False)
|
||||
iterator.chunk_type = "accumulated_json"
|
||||
|
||||
with patch.object(
|
||||
json.JSONDecoder, "raw_decode", autospec=True, side_effect=json.JSONDecoder.raw_decode
|
||||
) as spy:
|
||||
result = iterator._handle_accumulated_json_chunk(
|
||||
'{"type":"content_block_delta","index":0,"delta":'
|
||||
)
|
||||
with patch.object(json.JSONDecoder, "raw_decode", autospec=True, side_effect=json.JSONDecoder.raw_decode) as spy:
|
||||
result = iterator._handle_accumulated_json_chunk('{"type":"content_block_delta","index":0,"delta":')
|
||||
assert result is None
|
||||
assert spy.call_count == 0, "incomplete buffer should not be parsed"
|
||||
|
||||
|
|
@ -1482,21 +1469,15 @@ def test_accumulated_json_does_not_reparse_every_fragment():
|
|||
fragment.
|
||||
"""
|
||||
text = "x" * 200_000
|
||||
blob = json.dumps(
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}
|
||||
)
|
||||
blob = json.dumps({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}})
|
||||
fragments = [blob[i : i + 4096] for i in range(0, len(blob), 4096)]
|
||||
assert len(fragments) > 10, "need a multi-fragment payload to exercise the bug"
|
||||
|
||||
iterator = ModelResponseIterator(
|
||||
streaming_response=MagicMock(), sync_stream=True, json_mode=False
|
||||
)
|
||||
iterator = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=True, json_mode=False)
|
||||
iterator.chunk_type = "accumulated_json"
|
||||
|
||||
parsed = None
|
||||
with patch.object(
|
||||
json.JSONDecoder, "raw_decode", autospec=True, side_effect=json.JSONDecoder.raw_decode
|
||||
) as spy:
|
||||
with patch.object(json.JSONDecoder, "raw_decode", autospec=True, side_effect=json.JSONDecoder.raw_decode) as spy:
|
||||
for fragment in fragments:
|
||||
out = iterator._handle_accumulated_json_chunk(fragment)
|
||||
if out is not None:
|
||||
|
|
@ -1520,9 +1501,7 @@ def test_accumulated_json_concatenated_envelopes_do_not_wedge():
|
|||
and keeps the remainder, so both values surface across two calls.
|
||||
"""
|
||||
obj = '{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"a"}}'
|
||||
iterator = ModelResponseIterator(
|
||||
streaming_response=MagicMock(), sync_stream=True, json_mode=False
|
||||
)
|
||||
iterator = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=True, json_mode=False)
|
||||
iterator.chunk_type = "accumulated_json"
|
||||
|
||||
first = iterator._handle_accumulated_json_chunk(obj + obj)
|
||||
|
|
@ -1543,9 +1522,7 @@ def test_accumulated_json_heuristic_passes_but_value_still_incomplete():
|
|||
heuristic must let the parse attempt through, and pop_next_value
|
||||
finding nothing must propagate as None rather than raising.
|
||||
"""
|
||||
iterator = ModelResponseIterator(
|
||||
streaming_response=MagicMock(), sync_stream=True, json_mode=False
|
||||
)
|
||||
iterator = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=True, json_mode=False)
|
||||
iterator.chunk_type = "accumulated_json"
|
||||
|
||||
result = iterator._handle_accumulated_json_chunk('{"type": {"nested": 1}')
|
||||
|
|
@ -1559,9 +1536,7 @@ def test_accumulated_json_setter_and_sync_end_of_stream_drain():
|
|||
underlying stream ends, instead of being silently dropped.
|
||||
"""
|
||||
obj = '{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"a"}}'
|
||||
iterator = ModelResponseIterator(
|
||||
streaming_response=iter([]), sync_stream=True, json_mode=False
|
||||
)
|
||||
iterator = ModelResponseIterator(streaming_response=iter([]), sync_stream=True, json_mode=False)
|
||||
iterator.chunk_type = "accumulated_json"
|
||||
iterator.accumulated_json = obj # exercises the setter
|
||||
|
||||
|
|
@ -1576,9 +1551,7 @@ def test_accumulated_json_async_end_of_stream_drain():
|
|||
import asyncio
|
||||
|
||||
obj = '{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"a"}}'
|
||||
iterator = ModelResponseIterator(
|
||||
streaming_response=MagicMock(), sync_stream=False, json_mode=False
|
||||
)
|
||||
iterator = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=False, json_mode=False)
|
||||
iterator.chunk_type = "accumulated_json"
|
||||
iterator.accumulated_json = obj
|
||||
mock_async_iterator = MagicMock()
|
||||
|
|
@ -1772,7 +1745,9 @@ def test_web_search_calls_are_cumulative_through_incomplete_search():
|
|||
)
|
||||
|
||||
assert list(first_start.choices[0].delta.provider_specific_fields["web_search_calls"]) == ["srvtoolu_A"]
|
||||
assert first_result.choices[0].delta.provider_specific_fields["web_search_calls"]["srvtoolu_A"].status == "completed"
|
||||
assert (
|
||||
first_result.choices[0].delta.provider_specific_fields["web_search_calls"]["srvtoolu_A"].status == "completed"
|
||||
)
|
||||
calls = second_start.choices[0].delta.provider_specific_fields["web_search_calls"]
|
||||
assert list(calls) == ["srvtoolu_A", "srvtoolu_B"]
|
||||
assert calls["srvtoolu_A"].status == "completed"
|
||||
|
|
@ -2651,9 +2626,7 @@ class TestAnthropicChatCompletionPreCallLogging:
|
|||
calls = {"pre_call": []}
|
||||
logging_obj = MagicMock()
|
||||
logging_obj.pre_call.side_effect = lambda **kwargs: calls["pre_call"].append(kwargs)
|
||||
with patch.object(
|
||||
AnthropicConfig, "transform_request", return_value={"model": "m", "messages": []}
|
||||
):
|
||||
with patch.object(AnthropicConfig, "transform_request", return_value={"model": "m", "messages": []}):
|
||||
try:
|
||||
AnthropicChatCompletion().completion(**self._completion_kwargs(logging_obj=logging_obj))
|
||||
except Exception:
|
||||
|
|
@ -2739,6 +2712,272 @@ def test_served_model_reaches_assembled_stream_through_custom_stream_wrapper():
|
|||
assert assembled._hidden_params["provider_response_model"] == served_model
|
||||
|
||||
|
||||
_STREAMED_RISK_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": {"tier": {"type": "integer"}, "summary": {"type": "string"}},
|
||||
"required": ["tier", "summary"],
|
||||
}
|
||||
_STREAMED_RISK_OBJECT: Final = {"tier": 3, "summary": "Two-hour read-only window on the loyalty APIs"}
|
||||
_STREAMED_CONTACT_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": {"name": {"type": "string"}, "email": {"type": "string"}},
|
||||
"additionalProperties": False,
|
||||
}
|
||||
_STREAMED_BOOK_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"title": {"type": "string"}, "author": {"type": "string"}},
|
||||
"required": ["title", "author"],
|
||||
},
|
||||
},
|
||||
"required": ["name", "parameters"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
_STREAMED_CONTACT: Final = {"name": "Ada", "email": "ada@example.com"}
|
||||
_STREAMED_PERSON_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": {"name": {"type": "string"}, "age": {"type": "integer"}},
|
||||
"required": ["name", "age"],
|
||||
}
|
||||
_STREAMED_PERSON: Final = {"name": "Ada", "age": 36}
|
||||
|
||||
|
||||
def _json_tool_events(fragments: tuple[str, ...]) -> tuple[dict[str, object], ...]:
|
||||
return (
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-emulated-structured-output",
|
||||
"content": [],
|
||||
"stop_reason": None,
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 1},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 0,
|
||||
"content_block": {"type": "tool_use", "id": "toolu_1", "name": RESPONSE_FORMAT_TOOL_NAME, "input": {}},
|
||||
},
|
||||
*(
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": fragment}}
|
||||
for fragment in fragments
|
||||
),
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "tool_use", "stop_sequence": None},
|
||||
"usage": {"output_tokens": 20},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _streamed_json_mode_contents(
|
||||
fragments: tuple[str, ...], schema: dict[str, object] = _STREAMED_RISK_SCHEMA
|
||||
) -> tuple[str, ...]:
|
||||
iterator = ModelResponseIterator(
|
||||
streaming_response=MagicMock(),
|
||||
sync_stream=True,
|
||||
json_mode=True,
|
||||
response_format_schema=response_format_schema(schema),
|
||||
)
|
||||
chunks = tuple(iterator.chunk_parser(event) for event in _json_tool_events(fragments))
|
||||
assert all(chunk.choices[0].delta.tool_calls is None for chunk in chunks)
|
||||
assert chunks[-1].choices[0].finish_reason == "stop"
|
||||
return tuple(chunk.choices[0].delta.content or "" for chunk in chunks)
|
||||
|
||||
|
||||
def test_streamed_json_mode_envelope_is_unwrapped_when_the_block_ends():
|
||||
envelope = json.dumps({"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": _STREAMED_RISK_OBJECT})
|
||||
fragments = (envelope[:9], envelope[9:40], envelope[40:])
|
||||
|
||||
contents = _streamed_json_mode_contents(fragments)
|
||||
|
||||
assert json.loads("".join(contents)) == _STREAMED_RISK_OBJECT
|
||||
assert contents[:-2] == ("",) * (len(contents) - 2)
|
||||
|
||||
|
||||
def test_streamed_json_mode_schema_object_streams_as_it_arrives():
|
||||
answer = json.dumps(_STREAMED_RISK_OBJECT)
|
||||
fragments = (answer[:4], answer[4:12], answer[12:])
|
||||
|
||||
contents = _streamed_json_mode_contents(fragments)
|
||||
|
||||
assert "".join(contents) == answer
|
||||
assert contents[3] == answer[:12]
|
||||
assert contents[4] == answer[12:]
|
||||
assert contents[5] == ""
|
||||
|
||||
|
||||
def test_streamed_json_mode_truncated_output_is_passed_through_untouched():
|
||||
contents = _streamed_json_mode_contents(('{"na', 'me": "json_tool_call", "parameters": {"tier": 3'))
|
||||
|
||||
assert "".join(contents) == '{"name": "json_tool_call", "parameters": {"tier": 3'
|
||||
|
||||
|
||||
def test_streamed_json_mode_tool_name_envelope_is_unwrapped_when_name_is_a_schema_property():
|
||||
envelope = json.dumps({"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": _STREAMED_CONTACT})
|
||||
|
||||
contents = _streamed_json_mode_contents((envelope[:6], envelope[6:24], envelope[24:]), _STREAMED_CONTACT_SCHEMA)
|
||||
|
||||
assert json.loads("".join(contents)) == _STREAMED_CONTACT
|
||||
assert contents[:-2] == ("",) * (len(contents) - 2)
|
||||
|
||||
|
||||
def test_streamed_json_mode_name_property_answer_streams_once_its_name_value_is_known():
|
||||
answer = json.dumps(_STREAMED_CONTACT)
|
||||
|
||||
contents = _streamed_json_mode_contents((answer[:6], answer[6:24], answer[24:]), _STREAMED_CONTACT_SCHEMA)
|
||||
|
||||
assert "".join(contents) == answer
|
||||
assert contents[2] == ""
|
||||
assert contents[3] == answer[:24]
|
||||
assert contents[4] == answer[24:]
|
||||
|
||||
|
||||
def test_streamed_json_mode_partial_name_value_waits_for_the_value_to_close():
|
||||
answer = json.dumps({"name": "Ada"})
|
||||
|
||||
contents = _streamed_json_mode_contents((answer[:11], answer[11:]), _STREAMED_CONTACT_SCHEMA)
|
||||
|
||||
assert "".join(contents) == answer
|
||||
assert contents[2] == ""
|
||||
assert contents[3] == answer
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"schema, answer",
|
||||
[
|
||||
pytest.param(
|
||||
{"type": "object", "additionalProperties": {"type": "integer"}},
|
||||
{"alpha": 1, "beta": 2, "gamma": 3},
|
||||
id="schema_without_properties",
|
||||
),
|
||||
pytest.param(
|
||||
_STREAMED_RISK_SCHEMA,
|
||||
{"rev": 2, **_STREAMED_RISK_OBJECT},
|
||||
id="undeclared_first_key_holding_a_number",
|
||||
),
|
||||
pytest.param(
|
||||
{"type": "object", "properties": {"score": {"type": "integer"}}},
|
||||
{"note": "read-only window", "meta": {"score": 7}},
|
||||
id="string_first_key_on_a_schema_without_required_keys",
|
||||
),
|
||||
pytest.param(
|
||||
{"type": "object", "properties": {"summary": {"type": "string"}}, "additionalProperties": {"type": "object"}},
|
||||
{"user_1": {"summary": "likes tea", "visits": 3}},
|
||||
id="undeclared_key_the_schema_allows",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_streamed_json_mode_answer_that_cannot_be_an_envelope_streams_as_it_arrives(
|
||||
schema: dict[str, object], answer: dict[str, object]
|
||||
):
|
||||
encoded = json.dumps(answer)
|
||||
|
||||
contents = _streamed_json_mode_contents((encoded[:12], encoded[12:24], encoded[24:]), schema)
|
||||
|
||||
assert "".join(contents) == encoded
|
||||
assert contents[2:5] == (encoded[:12], encoded[12:24], encoded[24:])
|
||||
|
||||
|
||||
def test_streamed_json_mode_string_label_answer_streams_once_the_next_key_is_declared():
|
||||
answer = json.dumps({"note": "read-only window", **_STREAMED_RISK_OBJECT})
|
||||
|
||||
contents = _streamed_json_mode_contents((answer[:12], answer[12:40], answer[40:]))
|
||||
|
||||
assert "".join(contents) == answer
|
||||
assert contents[2] == ""
|
||||
assert contents[3] == answer[:40]
|
||||
assert contents[4] == answer[40:]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"envelope",
|
||||
[
|
||||
pytest.param(
|
||||
{"parameter_name": "RiskAssessment", "parameter_value": _STREAMED_RISK_OBJECT}, id="schema_name_label"
|
||||
),
|
||||
pytest.param(
|
||||
{"$FUNCTION_NAME": RESPONSE_FORMAT_TOOL_NAME, "PARAMETER_VALUE": _STREAMED_RISK_OBJECT},
|
||||
id="foreign_tool_name_key",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_streamed_json_mode_string_label_envelope_is_unwrapped_when_the_block_ends(envelope: dict[str, object]):
|
||||
encoded = json.dumps(envelope)
|
||||
|
||||
contents = _streamed_json_mode_contents((encoded[:12], encoded[12:40], encoded[40:]))
|
||||
|
||||
assert json.loads("".join(contents)) == _STREAMED_RISK_OBJECT
|
||||
assert contents[:-2] == ("",) * (len(contents) - 2)
|
||||
|
||||
|
||||
def test_streamed_json_mode_two_level_envelope_is_unwrapped_when_the_block_ends():
|
||||
envelope = json.dumps({"parameter": {"value": _STREAMED_RISK_OBJECT}})
|
||||
|
||||
contents = _streamed_json_mode_contents((envelope[:9], envelope[9:30], envelope[30:]))
|
||||
|
||||
assert json.loads("".join(contents)) == _STREAMED_RISK_OBJECT
|
||||
assert contents[:-2] == ("",) * (len(contents) - 2)
|
||||
|
||||
|
||||
def test_streamed_json_mode_accepted_answer_naming_the_tool_is_kept_whole():
|
||||
answer = json.dumps({"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": {"title": "Dune", "author": "Frank Herbert"}})
|
||||
|
||||
contents = _streamed_json_mode_contents((answer[:9], answer[9:40], answer[40:]), _STREAMED_BOOK_SCHEMA)
|
||||
|
||||
assert json.loads("".join(contents)) == json.loads(answer)
|
||||
assert contents[:-2] == ("",) * (len(contents) - 2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"answer, schema",
|
||||
[
|
||||
pytest.param(
|
||||
'{"name":"json_tool_call","parameters":{"title":"Dune","author":"Frank Herbert"}}',
|
||||
_STREAMED_BOOK_SCHEMA,
|
||||
id="answer_naming_the_tool",
|
||||
),
|
||||
pytest.param('{"city":"café 東京","country":"France"}', _STREAMED_RISK_SCHEMA, id="string_labels_only"),
|
||||
],
|
||||
)
|
||||
def test_streamed_json_mode_held_answer_without_an_envelope_keeps_the_model_text(
|
||||
answer: str, schema: dict[str, object]
|
||||
):
|
||||
contents = _streamed_json_mode_contents((answer[:9], answer[9:30], answer[30:]), schema)
|
||||
|
||||
assert "".join(contents) == answer
|
||||
assert contents[:-2] == ("",) * (len(contents) - 2)
|
||||
|
||||
|
||||
def test_envelope_verdict_holds_a_prefix_that_stays_undecided_past_the_limit():
|
||||
schema = response_format_schema(_STREAMED_RISK_SCHEMA)
|
||||
assert schema is not None
|
||||
never_closing_key = '{"' + "k" * 5000
|
||||
|
||||
assert _envelope_verdict(never_closing_key[:100], schema, RESPONSE_FORMAT_TOOL_NAME) == "undecided"
|
||||
assert _envelope_verdict(never_closing_key, schema, RESPONSE_FORMAT_TOOL_NAME) == "hold"
|
||||
|
||||
|
||||
def test_streamed_json_mode_answer_with_a_foreign_name_matches_the_non_streamed_answer():
|
||||
answer = {"name": "Person", "parameters": _STREAMED_PERSON}
|
||||
encoded = json.dumps(answer)
|
||||
schema = response_format_schema(_STREAMED_PERSON_SCHEMA)
|
||||
assert schema is not None
|
||||
|
||||
contents = _streamed_json_mode_contents((encoded[:16], encoded[16:34], encoded[34:]), _STREAMED_PERSON_SCHEMA)
|
||||
|
||||
assert "".join(contents) == encoded
|
||||
assert unwrap_response_format_envelope(answer, schema, RESPONSE_FORMAT_TOOL_NAME) == answer
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_anthropic_streaming_function_call_arguments_are_assembled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
@ -2784,9 +3023,7 @@ def test_anthropic_streaming_function_call_arguments_are_assembled(
|
|||
},
|
||||
{"type": "message_stop"},
|
||||
)
|
||||
body: Final = "".join(
|
||||
f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events
|
||||
)
|
||||
body: Final = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events)
|
||||
respx.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
|
|
@ -2820,18 +3057,12 @@ def test_anthropic_streaming_function_call_arguments_are_assembled(
|
|||
)
|
||||
chunks: Final = tuple(stream)
|
||||
tool_call_deltas: Final = tuple(
|
||||
chain.from_iterable(
|
||||
chunk.choices[0].delta.tool_calls or () for chunk in chunks if chunk.choices
|
||||
)
|
||||
chain.from_iterable(chunk.choices[0].delta.tool_calls or () for chunk in chunks if chunk.choices)
|
||||
)
|
||||
assert tuple(
|
||||
tool_call.function.name
|
||||
for tool_call in tool_call_deltas
|
||||
if tool_call.function.name is not None
|
||||
) == ("get_current_weather",)
|
||||
arguments: Final = "".join(
|
||||
tool_call.function.arguments or "" for tool_call in tool_call_deltas
|
||||
assert tuple(tool_call.function.name for tool_call in tool_call_deltas if tool_call.function.name is not None) == (
|
||||
"get_current_weather",
|
||||
)
|
||||
arguments: Final = "".join(tool_call.function.arguments or "" for tool_call in tool_call_deltas)
|
||||
assert json.loads(arguments) == {"location": "Boston", "unit": "fahrenheit"}
|
||||
assembled: Final = litellm.stream_chunk_builder(chunks=list(chunks), messages=messages)
|
||||
assert assembled.choices[0].message.tool_calls[0].function.name == "get_current_weather"
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messag
|
|||
from litellm.llms.anthropic.chat import ModelResponseIterator
|
||||
from litellm.llms.anthropic.common_utils import process_anthropic_headers
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
|
|
@ -36,7 +37,7 @@ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation im
|
|||
)
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
from litellm.types.llms.openai import ChatCompletionToolCallFunctionChunk
|
||||
from litellm.types.utils import ChatCompletionToolCallChunk, ServerToolUse, Usage
|
||||
from litellm.types.utils import ChatCompletionToolCallChunk, ModelResponse, ServerToolUse, Usage
|
||||
|
||||
|
||||
def test_response_format_transformation_unit_test():
|
||||
|
|
@ -7295,6 +7296,412 @@ def test_convert_tool_response_to_message_no_arguments():
|
|||
assert message is None
|
||||
|
||||
|
||||
_RISK_SCHEMA: Final = {
|
||||
"$defs": {"Owner": {"title": "Owner", "type": "object", "properties": {"team": {"type": "string"}}}},
|
||||
"title": "RiskAssessment",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tier": {"title": "Tier", "type": "integer"},
|
||||
"summary": {"title": "Summary", "type": "string"},
|
||||
"owner": {"$ref": "#/$defs/Owner"},
|
||||
},
|
||||
"required": ["tier", "summary"],
|
||||
}
|
||||
_RISK_OBJECT: Final = {"tier": 3, "summary": "Two-hour read-only window on the loyalty APIs"}
|
||||
_PERSON_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": {"name": {"type": "string"}, "age": {"type": "integer"}},
|
||||
"required": ["name", "age"],
|
||||
}
|
||||
_OPTIONAL_FIELDS_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": {"note": {"type": "string"}, "score": {"type": "integer"}},
|
||||
}
|
||||
_TREE_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": {"value": {"type": "integer"}, "child": {"type": "object"}},
|
||||
"required": ["value"],
|
||||
}
|
||||
_OPEN_SCORE_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": {"score": {"type": "integer"}},
|
||||
"additionalProperties": True,
|
||||
}
|
||||
_VALUES_SCHEMA: Final = {"type": "object", "properties": {"values": {"type": "array"}}, "required": ["values"]}
|
||||
_REMOTE_REF_SCHEMA: Final = {**_PERSON_SCHEMA, "$ref": "https://schemas.example.com/person.json"}
|
||||
_LOOSE_SCHEMA: Final = {"type": "object", "properties": {"name": {"type": "string"}, "email": {"type": "string"}}}
|
||||
_CLOSED_CONTACT_SCHEMA: Final = {**_LOOSE_SCHEMA, "additionalProperties": False}
|
||||
_CONTACT: Final = {"name": "Ada", "email": "ada@example.com"}
|
||||
_NAMED_BOOK_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"title": {"type": "string"}, "author": {"type": "string"}},
|
||||
"required": ["title", "author"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
"required": ["name", "parameters"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
_NAMED_BOOK: Final = {"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": {"title": "Dune", "author": "Frank Herbert"}}
|
||||
_RECURSIVE_NODE_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "string"},
|
||||
"title": {"type": "string"},
|
||||
"parent": {"anyOf": [{"$ref": "#"}, {"type": "null"}]},
|
||||
},
|
||||
"required": ["id", "title"],
|
||||
}
|
||||
_CHILD_WITHOUT_TITLE: Final = {"id": "child-7", "parent": {"id": "root-1", "title": "Root", "parent": None}}
|
||||
_JSON_OBJECT_MODE_SCHEMA: Final = {"type": "object", "properties": {}, "additionalProperties": True}
|
||||
_SUMMARY_PROPERTIES: Final = {"summary": {"type": "string"}}
|
||||
_PER_USER_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": _SUMMARY_PROPERTIES,
|
||||
"additionalProperties": {"type": "object"},
|
||||
}
|
||||
_PER_USER_PATTERN_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": _SUMMARY_PROPERTIES,
|
||||
"patternProperties": {"^user_": {"type": "object"}},
|
||||
}
|
||||
_PER_USER_ANSWER: Final = {"user_1": {"summary": "likes tea", "visits": 3}}
|
||||
|
||||
|
||||
def _schema(document: dict[str, object]) -> ResponseFormatSchema:
|
||||
schema = response_format_schema(document)
|
||||
assert schema is not None
|
||||
return schema
|
||||
|
||||
|
||||
def _json_tool_call(arguments: object) -> list[ChatCompletionToolCallChunk]:
|
||||
return [
|
||||
ChatCompletionToolCallChunk(
|
||||
id="toolu_1",
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=RESPONSE_FORMAT_TOOL_NAME, arguments=json.dumps(arguments)
|
||||
),
|
||||
index=0,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"schema, tool_input, expected_content",
|
||||
[
|
||||
pytest.param(
|
||||
_RISK_SCHEMA,
|
||||
{"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": _RISK_OBJECT},
|
||||
_RISK_OBJECT,
|
||||
id="name_and_parameters_envelope",
|
||||
),
|
||||
pytest.param(_RISK_SCHEMA, {"input": _RISK_OBJECT}, _RISK_OBJECT, id="input_wrapper"),
|
||||
pytest.param(_RISK_SCHEMA, {"arguments": _RISK_OBJECT}, _RISK_OBJECT, id="arguments_wrapper"),
|
||||
pytest.param(
|
||||
_PERSON_SCHEMA,
|
||||
{"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": {"name": "Ada", "age": 36}},
|
||||
{"name": "Ada", "age": 36},
|
||||
id="envelope_key_shared_with_a_schema_property",
|
||||
),
|
||||
pytest.param(
|
||||
_OPTIONAL_FIELDS_SCHEMA,
|
||||
{"input": {"score": 7}},
|
||||
{"score": 7},
|
||||
id="wrapper_around_an_all_optional_schema",
|
||||
),
|
||||
pytest.param(
|
||||
_RISK_SCHEMA,
|
||||
{"parameter": {"value": _RISK_OBJECT}},
|
||||
_RISK_OBJECT,
|
||||
id="two_level_envelope",
|
||||
),
|
||||
pytest.param(
|
||||
_RISK_SCHEMA,
|
||||
{"parameter_name": "RiskAssessment", "parameter_value": _RISK_OBJECT},
|
||||
_RISK_OBJECT,
|
||||
id="string_label_beside_the_wrapper",
|
||||
),
|
||||
pytest.param(
|
||||
_RISK_SCHEMA,
|
||||
{"$FUNCTION_NAME": RESPONSE_FORMAT_TOOL_NAME, "PARAMETER_VALUE": _RISK_OBJECT},
|
||||
_RISK_OBJECT,
|
||||
id="foreign_tool_name_key_beside_the_wrapper",
|
||||
),
|
||||
pytest.param(
|
||||
_RISK_SCHEMA,
|
||||
{"note": "read-only window", **_RISK_OBJECT},
|
||||
{"note": "read-only window", **_RISK_OBJECT},
|
||||
id="string_label_beside_schema_keys",
|
||||
),
|
||||
pytest.param(
|
||||
{**_RISK_SCHEMA, "required": "tier"},
|
||||
{"parameter_name": "RiskAssessment", "parameter_value": _RISK_OBJECT},
|
||||
{"parameter_name": "RiskAssessment", "parameter_value": _RISK_OBJECT},
|
||||
id="string_label_when_required_is_malformed",
|
||||
),
|
||||
pytest.param(
|
||||
_RISK_SCHEMA,
|
||||
{"name": RESPONSE_FORMAT_TOOL_NAME, "arguments": json.dumps(_RISK_OBJECT)},
|
||||
_RISK_OBJECT,
|
||||
id="tool_name_envelope_holding_encoded_json",
|
||||
),
|
||||
pytest.param(
|
||||
_RISK_SCHEMA,
|
||||
{"tier": 1, "parameters": _RISK_OBJECT},
|
||||
{"tier": 1, "parameters": _RISK_OBJECT},
|
||||
id="schema_key_beside_a_wrapper_key",
|
||||
),
|
||||
pytest.param(
|
||||
_RECURSIVE_NODE_SCHEMA,
|
||||
_CHILD_WITHOUT_TITLE,
|
||||
_CHILD_WITHOUT_TITLE,
|
||||
id="recursive_schema_keeps_the_answer_over_its_complete_parent",
|
||||
),
|
||||
pytest.param(
|
||||
{**_RISK_SCHEMA, "$schema": 5},
|
||||
{"input": _RISK_OBJECT},
|
||||
_RISK_OBJECT,
|
||||
id="non_string_dollar_schema",
|
||||
),
|
||||
pytest.param(
|
||||
_OPEN_SCORE_SCHEMA,
|
||||
{"meta": {"score": 7}, "label": "keep"},
|
||||
{"meta": {"score": 7}, "label": "keep"},
|
||||
id="valid_answer_with_extra_keys_and_a_schema_shaped_nested_object",
|
||||
),
|
||||
pytest.param(_VALUES_SCHEMA, {"values": [1, 2]}, {"values": [1, 2]}, id="values_is_a_schema_property"),
|
||||
pytest.param(
|
||||
_REMOTE_REF_SCHEMA,
|
||||
{"input": {"name": "Ada", "age": 36}},
|
||||
{"name": "Ada", "age": 36},
|
||||
id="remote_ref_schema_falls_back_to_key_matching",
|
||||
),
|
||||
pytest.param(
|
||||
_CLOSED_CONTACT_SCHEMA,
|
||||
{"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": _CONTACT},
|
||||
_CONTACT,
|
||||
id="tool_name_envelope_when_the_schema_lists_no_required_keys",
|
||||
),
|
||||
pytest.param(
|
||||
_LOOSE_SCHEMA,
|
||||
{"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": _CONTACT},
|
||||
_CONTACT,
|
||||
id="tool_name_envelope_on_an_open_schema",
|
||||
),
|
||||
pytest.param(_NAMED_BOOK_SCHEMA, _NAMED_BOOK, _NAMED_BOOK, id="accepted_answer_whose_name_is_the_tool_name"),
|
||||
pytest.param(
|
||||
_LOOSE_SCHEMA,
|
||||
{"name": RESPONSE_FORMAT_TOOL_NAME, "email": "ada@example.com"},
|
||||
{"name": RESPONSE_FORMAT_TOOL_NAME, "email": "ada@example.com"},
|
||||
id="tool_name_value_without_a_nested_object_stays",
|
||||
),
|
||||
pytest.param(_RISK_SCHEMA, _RISK_OBJECT, _RISK_OBJECT, id="plain_schema_object"),
|
||||
pytest.param(
|
||||
_TREE_SCHEMA,
|
||||
{"value": 1, "child": {"value": 2}},
|
||||
{"value": 1, "child": {"value": 2}},
|
||||
id="schema_object_whose_property_holds_a_schema_shaped_object",
|
||||
),
|
||||
pytest.param(_RISK_SCHEMA, {"input": {"tier": 3}}, {"tier": 3}, id="inner_object_missing_required_key"),
|
||||
pytest.param(_RISK_SCHEMA, {"values": {"tier": 2}}, {"tier": 2}, id="values_wrapper_missing_required_key"),
|
||||
pytest.param(
|
||||
_RISK_SCHEMA,
|
||||
{"first": _RISK_OBJECT, "second": _RISK_OBJECT},
|
||||
{"first": _RISK_OBJECT, "second": _RISK_OBJECT},
|
||||
id="two_candidate_objects",
|
||||
),
|
||||
pytest.param(
|
||||
_OPTIONAL_FIELDS_SCHEMA,
|
||||
{"meta": {"source": "change-request"}},
|
||||
{"meta": {"source": "change-request"}},
|
||||
id="inner_object_without_schema_keys",
|
||||
),
|
||||
pytest.param(
|
||||
_JSON_OBJECT_MODE_SCHEMA,
|
||||
{"input": _RISK_OBJECT},
|
||||
{"input": _RISK_OBJECT},
|
||||
id="json_object_mode_without_schema_properties",
|
||||
),
|
||||
pytest.param(
|
||||
_JSON_OBJECT_MODE_SCHEMA,
|
||||
{"values": _RISK_OBJECT},
|
||||
_RISK_OBJECT,
|
||||
id="json_object_mode_values_wrapper",
|
||||
),
|
||||
pytest.param(
|
||||
_JSON_OBJECT_MODE_SCHEMA,
|
||||
{"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": _RISK_OBJECT},
|
||||
_RISK_OBJECT,
|
||||
id="json_object_mode_tool_name_envelope",
|
||||
),
|
||||
pytest.param(_PER_USER_SCHEMA, _PER_USER_ANSWER, _PER_USER_ANSWER, id="key_allowed_by_additional_properties"),
|
||||
pytest.param(
|
||||
_PER_USER_PATTERN_SCHEMA, _PER_USER_ANSWER, _PER_USER_ANSWER, id="key_allowed_by_pattern_properties"
|
||||
),
|
||||
pytest.param(_OPEN_SCORE_SCHEMA, {"input": {"score": 7}}, {"input": {"score": 7}}, id="key_allowed_by_true"),
|
||||
pytest.param(
|
||||
{**_PER_USER_SCHEMA, "required": ["summary"]},
|
||||
{"input": {"summary": "likes tea"}},
|
||||
{"summary": "likes tea"},
|
||||
id="allowed_key_on_a_schema_with_required_keys",
|
||||
),
|
||||
pytest.param(
|
||||
_PER_USER_SCHEMA,
|
||||
{"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": {"summary": "likes tea"}},
|
||||
{"summary": "likes tea"},
|
||||
id="tool_name_envelope_on_a_schema_allowing_extra_keys",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_convert_tool_response_to_message_unwraps_only_an_unambiguous_schema_envelope(
|
||||
schema: dict, tool_input: dict, expected_content: dict
|
||||
):
|
||||
message = AnthropicConfig.convert_tool_response_to_message(
|
||||
tool_calls=_json_tool_call(tool_input),
|
||||
response_format_schema=_schema(schema),
|
||||
)
|
||||
|
||||
assert message is not None
|
||||
assert json.loads(message.content) == expected_content
|
||||
|
||||
|
||||
_TOO_DEEP_TO_DECODE: Final = "[" * 100_000 + "]" * 100_000
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"arguments",
|
||||
[json.dumps({"name": RESPONSE_FORMAT_TOOL_NAME, "arguments": _TOO_DEEP_TO_DECODE}), _TOO_DEEP_TO_DECODE],
|
||||
ids=["encoded_inner_object", "whole_arguments"],
|
||||
)
|
||||
def test_convert_tool_response_to_message_keeps_arguments_too_deep_to_decode(arguments: str):
|
||||
message = AnthropicConfig.convert_tool_response_to_message(
|
||||
tool_calls=[
|
||||
ChatCompletionToolCallChunk(
|
||||
id="toolu_1",
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(name=RESPONSE_FORMAT_TOOL_NAME, arguments=arguments),
|
||||
index=0,
|
||||
)
|
||||
],
|
||||
response_format_schema=_schema(_RISK_SCHEMA),
|
||||
)
|
||||
|
||||
assert message is not None
|
||||
assert message.content == arguments
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"document",
|
||||
[
|
||||
{"type": "object", "properties": ["tier", "summary"]},
|
||||
{"type": "object", "properties": "tier"},
|
||||
["tier", "summary"],
|
||||
],
|
||||
ids=["properties_list", "properties_string", "schema_list"],
|
||||
)
|
||||
def test_response_format_schema_rejects_a_malformed_schema(document: object):
|
||||
assert response_format_schema(document) is None
|
||||
|
||||
|
||||
def test_json_mode_with_user_tool_call_unwraps_the_envelope_into_content():
|
||||
user_tool_call: Final = ChatCompletionToolCallChunk(
|
||||
id="toolu_user",
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(name="lookup_change_window", arguments='{"system": "ledger"}'),
|
||||
index=1,
|
||||
)
|
||||
|
||||
replacement, remaining_tool_calls, extra_content = AnthropicConfig()._resolve_json_mode_non_streaming(
|
||||
json_mode=True,
|
||||
tool_calls=[*_json_tool_call({"input": _RISK_OBJECT}), user_tool_call],
|
||||
response_format_schema=_schema(_RISK_SCHEMA),
|
||||
)
|
||||
|
||||
assert replacement is None
|
||||
assert remaining_tool_calls == [user_tool_call]
|
||||
assert extra_content is not None
|
||||
assert json.loads(extra_content) == _RISK_OBJECT
|
||||
|
||||
|
||||
def test_vertex_claude_tool_emulated_response_format_returns_the_object_the_model_wrapped():
|
||||
config = VertexAIAnthropicConfig()
|
||||
optional_params = config.map_openai_params(
|
||||
non_default_params={
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "lookup_change_window",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"system": {"type": "string"}},
|
||||
"required": ["system"],
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "RiskAssessment", "schema": _RISK_SCHEMA},
|
||||
},
|
||||
},
|
||||
optional_params={},
|
||||
model="claude-emulated-structured-output",
|
||||
drop_params=False,
|
||||
)
|
||||
json_mode = optional_params.pop("json_mode")
|
||||
request = config.transform_request(
|
||||
model="claude-emulated-structured-output",
|
||||
messages=[{"role": "user", "content": "Assess this change request"}],
|
||||
optional_params={**optional_params, "is_vertex_request": True},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
json_tool = next(tool for tool in request["tools"] if tool["name"] == RESPONSE_FORMAT_TOOL_NAME)
|
||||
assert json_tool["input_schema"]["required"] == ["tier", "summary"]
|
||||
assert "output_format" not in request
|
||||
assert "output_config" not in request
|
||||
completion_response = {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-emulated-structured-output",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_1",
|
||||
"name": RESPONSE_FORMAT_TOOL_NAME,
|
||||
"input": {"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": _RISK_OBJECT},
|
||||
}
|
||||
],
|
||||
"stop_reason": "tool_use",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 12, "output_tokens": 30},
|
||||
}
|
||||
|
||||
result = config.transform_response(
|
||||
model="claude-emulated-structured-output",
|
||||
raw_response=httpx.Response(200, json=completion_response),
|
||||
model_response=ModelResponse(),
|
||||
logging_obj=MagicMock(),
|
||||
request_data=request,
|
||||
messages=[{"role": "user", "content": "Assess this change request"}],
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
json_mode=json_mode,
|
||||
)
|
||||
|
||||
assert json.loads(result.choices[0].message.content) == _RISK_OBJECT
|
||||
assert result.choices[0].message.tool_calls is None
|
||||
assert result.choices[0].finish_reason == "stop"
|
||||
|
||||
|
||||
def test_anthropic_tool_with_image():
|
||||
import json
|
||||
|
||||
|
|
@ -7606,9 +8013,7 @@ async def test_anthropic_api_max_completion_tokens(model: str):
|
|||
print("request_body: ", request_body)
|
||||
|
||||
assert request_body == {
|
||||
"messages": [
|
||||
{"role": "user", "content": [{"type": "text", "text": "Hello!"}]}
|
||||
],
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello!"}]}],
|
||||
"max_tokens": 10,
|
||||
"model": model.split("/")[-1],
|
||||
}
|
||||
|
|
@ -7826,9 +8231,7 @@ async def test_anthropic_api_prompt_caching_with_content_str():
|
|||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
]
|
||||
translated_system_message = litellm.AnthropicConfig().translate_system_message(
|
||||
messages=system_message
|
||||
)
|
||||
translated_system_message = litellm.AnthropicConfig().translate_system_message(messages=system_message)
|
||||
|
||||
assert translated_system_message == [
|
||||
# System Message
|
||||
|
|
@ -7898,9 +8301,9 @@ async def test_anthropic_api_prompt_caching_with_content_str():
|
|||
|
||||
assert len(translated_messages) == len(expected_messages)
|
||||
for idx, i in enumerate(translated_messages):
|
||||
assert (
|
||||
i == expected_messages[idx]
|
||||
), "Error on idx={}. Got={}, Expected={}".format(idx, i, expected_messages[idx])
|
||||
assert i == expected_messages[idx], "Error on idx={}. Got={}, Expected={}".format(
|
||||
idx, i, expected_messages[idx]
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
|
|||
|
|
@ -20,8 +20,10 @@ from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
|||
from litellm.llms.bedrock.chat.invoke_handler import (
|
||||
AmazonOpenAICompatibleStreamDecoder,
|
||||
AWSEventStreamDecoder,
|
||||
_stream_decoder,
|
||||
make_call,
|
||||
make_sync_call,
|
||||
response_format_schema_from_request,
|
||||
)
|
||||
from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_stream_event_statuses
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
|
|
@ -1618,6 +1620,91 @@ def test_invoke_decoder_reports_only_missing_aws_dependency(missing):
|
|||
assert error.value is failure
|
||||
|
||||
|
||||
_INVOKE_RISK_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": {"tier": {"type": "integer"}, "summary": {"type": "string"}},
|
||||
"required": ["tier", "summary"],
|
||||
}
|
||||
_INVOKE_RISK_OBJECT: Final = {"tier": 2, "summary": "Weekend database migration with a tested rollback"}
|
||||
|
||||
|
||||
def _invoke_json_mode_request(tools: Sequence[Mapping[str, object]]) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"anthropic_version": "bedrock-2023-05-31",
|
||||
"messages": [{"role": "user", "content": "Assess this change request"}],
|
||||
"tools": list(tools),
|
||||
"tool_choice": {"type": "tool", "name": "json_tool_call"},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _invoke_json_tool_events(fragments: Sequence[str]) -> tuple[dict[str, object], ...]:
|
||||
deltas = tuple(
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": fragment}}
|
||||
for fragment in fragments
|
||||
)
|
||||
return (
|
||||
{"type": "message_start", "message": {"id": "msg_1", "usage": {"input_tokens": 1, "output_tokens": 0}}},
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 0,
|
||||
"content_block": {"type": "tool_use", "id": "t1", "name": "json_tool_call", "input": {}},
|
||||
},
|
||||
*deltas,
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{"type": "message_delta", "delta": {"stop_reason": "tool_use"}, "usage": {"output_tokens": 9}},
|
||||
)
|
||||
|
||||
|
||||
def _invoke_streamed_json_mode_contents(data: str, fragments: Sequence[str], json_mode: bool = True) -> tuple[str, ...]:
|
||||
decoder = _stream_decoder(
|
||||
"anthropic",
|
||||
model="anthropic.claude-sonnet-4",
|
||||
json_mode=True,
|
||||
sync_stream=True,
|
||||
response_format_schema=response_format_schema_from_request(data, json_mode),
|
||||
)
|
||||
chunks = tuple(decoder._chunk_parser(event) for event in _invoke_json_tool_events(fragments))
|
||||
assert all(not chunk.choices[0].delta.tool_calls for chunk in chunks)
|
||||
return tuple(chunk.choices[0].delta.content or "" for chunk in chunks)
|
||||
|
||||
|
||||
def test_invoke_streamed_json_mode_envelope_is_unwrapped_with_the_schema_from_the_request_body():
|
||||
envelope: Final = json.dumps({"tool_name": "json_tool_call", "parameters": _INVOKE_RISK_OBJECT})
|
||||
data: Final = _invoke_json_mode_request([{"name": "json_tool_call", "input_schema": _INVOKE_RISK_SCHEMA}])
|
||||
|
||||
contents = _invoke_streamed_json_mode_contents(data, (envelope[:17], envelope[17:48], envelope[48:]))
|
||||
|
||||
assert json.loads("".join(contents)) == _INVOKE_RISK_OBJECT
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("data", "json_mode"),
|
||||
[
|
||||
pytest.param("not json", True, id="unparseable_body"),
|
||||
pytest.param(json.dumps([1, 2]), True, id="body_is_not_an_object"),
|
||||
pytest.param(
|
||||
_invoke_json_mode_request([{"name": "other_tool", "input_schema": _INVOKE_RISK_SCHEMA}]),
|
||||
True,
|
||||
id="no_json_tool",
|
||||
),
|
||||
pytest.param(
|
||||
_invoke_json_mode_request([{"name": "json_tool_call", "input_schema": _INVOKE_RISK_SCHEMA}]),
|
||||
False,
|
||||
id="json_mode_off",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_invoke_stream_keeps_the_wrapper_when_the_request_body_carries_no_json_tool_schema(data: str, json_mode: bool):
|
||||
envelope: Final = json.dumps({"tool_name": "json_tool_call", "parameters": _INVOKE_RISK_OBJECT})
|
||||
|
||||
assert response_format_schema_from_request(data, json_mode) is None
|
||||
contents = _invoke_streamed_json_mode_contents(data, (envelope[:17], envelope[17:]), json_mode)
|
||||
|
||||
assert json.loads("".join(contents)) == {"tool_name": "json_tool_call", "parameters": _INVOKE_RISK_OBJECT}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_converse_stream_thinking_merges_into_content_with_think_tags() -> None:
|
||||
frames: Final = (
|
||||
|
|
|
|||
|
|
@ -514,6 +514,29 @@ def test_messages_request_strips_effort_for_haiku_45():
|
|||
assert opus_result["output_config"] == {"effort": "high"}
|
||||
|
||||
|
||||
def test_messages_request_moves_output_format_into_output_config():
|
||||
config = VertexAIPartnerModelsAnthropicMessagesConfig()
|
||||
output_format: Final = {
|
||||
"type": "json_schema",
|
||||
"schema": {"type": "object", "properties": {"answer": {"type": "integer"}}, "required": ["answer"]},
|
||||
}
|
||||
|
||||
result = config.transform_anthropic_messages_request(
|
||||
model="claude-opus-4-6",
|
||||
messages=[{"role": "user", "content": "What is 6 times 7?"}],
|
||||
anthropic_messages_optional_request_params={
|
||||
"max_tokens": 1024,
|
||||
"output_format": output_format,
|
||||
"output_config": {"effort": "high"},
|
||||
},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "output_format" not in result
|
||||
assert result["output_config"] == {"effort": "high", "format": output_format}
|
||||
|
||||
|
||||
def test_provider_config_manager_reuses_vertex_anthropic_messages_config_instance():
|
||||
"""
|
||||
Regression test: repeated provider config lookups for the same Vertex Claude model
|
||||
|
|
|
|||
|
|
@ -252,9 +252,8 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix():
|
|||
This test verifies that:
|
||||
1. Claude Sonnet 4.5 uses tool-based structured outputs when ``response_format``
|
||||
is given to the OpenAI-compat path (the path that triggered #18625).
|
||||
2. ``output_format`` is forwarded to Vertex AI when present — Vertex now
|
||||
accepts the field; the prior blanket-strip behavior was the silent drop
|
||||
of Anthropic Structured Outputs that this PR fixes.
|
||||
2. ``output_format`` reaches Vertex AI as ``output_config.format``, since
|
||||
Vertex rejects the top-level field as deprecated.
|
||||
"""
|
||||
config = VertexAIAnthropicConfig()
|
||||
|
||||
|
|
@ -340,12 +339,8 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix():
|
|||
headers={},
|
||||
)
|
||||
|
||||
# output_format is now forwarded to Vertex (Vertex parity has shifted —
|
||||
# it accepts the field and uses it to enforce the JSON schema). The
|
||||
# prior behavior silently stripped it, hiding Structured Outputs from
|
||||
# callers who explicitly requested them.
|
||||
assert "output_format" in final_data
|
||||
assert final_data["output_format"]["type"] == "json_schema"
|
||||
assert "output_format" not in final_data
|
||||
assert final_data["output_config"]["format"]["type"] == "json_schema"
|
||||
assert (
|
||||
"model" not in final_data
|
||||
), "model is still stripped (Vertex routes by URL)"
|
||||
|
|
@ -735,7 +730,7 @@ def test_vertex_ai_anthropic_output_config_non_dict_dropped():
|
|||
|
||||
|
||||
def test_vertex_ai_anthropic_output_format_and_output_config_effort_preserved():
|
||||
"""Both ``output_format`` and ``output_config.effort`` are forwarded on Vertex 4.6/4.7."""
|
||||
"""``output_format`` is folded into ``output_config`` next to ``effort`` on Vertex 4.6/4.7."""
|
||||
config = VertexAIAnthropicConfig()
|
||||
messages = [{"role": "user", "content": "Extract structured data"}]
|
||||
|
||||
|
|
@ -782,10 +777,8 @@ def test_vertex_ai_anthropic_output_format_and_output_config_effort_preserved():
|
|||
headers={},
|
||||
)
|
||||
|
||||
# output_format flows through unchanged — Vertex AI Claude accepts it.
|
||||
assert result["output_format"] == output_format
|
||||
# output_config.effort now flows through (Vertex accepts it on 4.6/4.7).
|
||||
assert result["output_config"] == {"effort": "high"}
|
||||
assert "output_format" not in result
|
||||
assert result["output_config"] == {"effort": "high", "format": output_format}
|
||||
assert result["max_tokens"] == 2048
|
||||
assert "model" not in result, "model is still stripped (Vertex routes by URL)"
|
||||
finally:
|
||||
|
|
@ -859,6 +852,97 @@ def test_sanitize_strips_effort_for_haiku_45():
|
|||
assert data["output_config"] == {"effort": "high"}
|
||||
|
||||
|
||||
_STRUCTURED_FORMAT: Final = {
|
||||
"type": "json_schema",
|
||||
"schema": {"type": "object", "properties": {"result": {"type": "string"}}, "required": ["result"]},
|
||||
}
|
||||
_CALLER_FORMAT: Final = {
|
||||
"type": "json_schema",
|
||||
"schema": {"type": "object", "properties": {"answer": {"type": "integer"}}},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, data, expected_output_config",
|
||||
[
|
||||
pytest.param(
|
||||
"claude-opus-4-6",
|
||||
{"output_format": _STRUCTURED_FORMAT},
|
||||
{"format": _STRUCTURED_FORMAT},
|
||||
id="output_format_alone",
|
||||
),
|
||||
pytest.param(
|
||||
"claude-opus-4-6",
|
||||
{"output_format": _STRUCTURED_FORMAT, "output_config": {"effort": "high"}},
|
||||
{"effort": "high", "format": _STRUCTURED_FORMAT},
|
||||
id="merged_with_effort",
|
||||
),
|
||||
pytest.param(
|
||||
"claude-haiku-4-5@20251001",
|
||||
{"output_format": _STRUCTURED_FORMAT, "output_config": {"effort": "high"}},
|
||||
{"format": _STRUCTURED_FORMAT},
|
||||
id="effort_still_dropped_for_haiku_45",
|
||||
),
|
||||
pytest.param(
|
||||
"claude-opus-4-6",
|
||||
{"output_format": _STRUCTURED_FORMAT, "output_config": {"format": _CALLER_FORMAT}},
|
||||
{"format": _STRUCTURED_FORMAT},
|
||||
id="output_format_wins_over_output_config_format",
|
||||
),
|
||||
pytest.param(
|
||||
"claude-opus-4-6",
|
||||
{"output_format": _STRUCTURED_FORMAT, "output_config": {"format": None}},
|
||||
{"format": _STRUCTURED_FORMAT},
|
||||
id="null_output_config_format_counts_as_absent",
|
||||
),
|
||||
pytest.param(
|
||||
"claude-opus-4-6",
|
||||
{"output_format": _STRUCTURED_FORMAT, "output_config": "not-a-dict"},
|
||||
{"format": _STRUCTURED_FORMAT},
|
||||
id="malformed_output_config_replaced",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_sanitize_folds_output_format_into_output_config(model: str, data: dict, expected_output_config: dict):
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.output_params_utils import (
|
||||
sanitize_vertex_anthropic_output_params,
|
||||
)
|
||||
|
||||
request: Final = copy.deepcopy(data)
|
||||
sanitize_vertex_anthropic_output_params(request, model)
|
||||
|
||||
assert "output_format" not in request
|
||||
assert request["output_config"] == expected_output_config
|
||||
|
||||
|
||||
def test_vertex_ai_native_response_format_reaches_vertex_as_output_config_format(local_model_cost_map):
|
||||
config = VertexAIAnthropicConfig()
|
||||
schema: Final = {"type": "object", "properties": {"result": {"type": "string"}}, "required": ["result"]}
|
||||
optional_params: Final = config.map_openai_params(
|
||||
non_default_params={
|
||||
"response_format": {"type": "json_schema", "json_schema": {"name": "result", "schema": schema}},
|
||||
"reasoning_effort": "high",
|
||||
},
|
||||
optional_params={},
|
||||
model="claude-fable-5-1",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
request: Final = config.transform_request(
|
||||
model="claude-fable-5-1",
|
||||
messages=[{"role": "user", "content": "Return the result"}],
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "output_format" not in request
|
||||
assert request["output_config"]["format"] == optional_params["output_format"]
|
||||
assert request["output_config"]["format"]["schema"]["properties"] == schema["properties"]
|
||||
assert request["output_config"]["effort"] == optional_params["output_config"]["effort"]
|
||||
assert "tool_choice" not in request
|
||||
|
||||
|
||||
def test_vertex_ai_fable_5_1_response_format_uses_native_output_format(local_model_cost_map):
|
||||
"""Regression: Fable 5.1 rejects forced tool use, so the vertex map entry
|
||||
advertises native structured output and ``response_format`` must map to
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue