diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index fa519740e88..c05407b7972 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -32,6 +32,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( tool_call_ids_answered_by_client, tool_call_ids_answered_in, ) +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import string_field from litellm.llms.anthropic.common_utils import ( is_claude_code_one_shot_subagent_request, supports_anthropic_cache_control, @@ -128,7 +129,7 @@ def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool: return False return ( not isinstance(fields.get("content"), list) - or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT + or string_field(fields, "role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT ) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 5fbd787b3b6..938ab585b72 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -8,7 +8,7 @@ import io import json import mimetypes import re -from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from graphlib import CycleError, TopologicalSorter from itertools import chain, groupby, islice @@ -3235,3 +3235,15 @@ def text_completion_prompt_to_messages(prompt: object) -> tuple[AllMessageValues if string_entries and len(string_entries) == len(entries): return tuple(ChatCompletionUserMessage(role="user", content=entry) for entry in string_entries) raise ValueError(f"`prompt` must be a non-empty string or a non-empty list of strings. Got: {prompt_type_name}.") + + +def is_present_non_string(value: object) -> bool: + return value is not None and not isinstance(value, str) + + +def invalid_type_message(field_path: str, value: object) -> str: + return f"Invalid type for '{field_path}': expected a string, got {type(value).__name__}" + + +def is_unhashable_value_behind(error: TypeError, value: object) -> bool: + return not isinstance(value, Hashable) and f"unhashable type: '{type(value).__name__}'" in str(error) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 320b4df8192..98348e434b2 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -32,6 +32,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import parts_of, place_mid_conversation_system, split_leading_system_run, + string_field, ) from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client, header_value @@ -2023,7 +2024,7 @@ def _server_tool_answers_replaced_after(messages: Sequence[object], index: int) following: Final = _messages_until_the_next_assistant(messages, index) a_user_turn_remains: Final = any( message_field(message, "role") in ("user", "tool", "function") - and message_field(message, "tool_call_id") not in kept_server_result_ids + and string_field(message, "tool_call_id") not in kept_server_result_ids for message in following ) return _tool_call_ids_in_tool_messages(following) & kept_server_result_ids if a_user_turn_remains else frozenset() diff --git a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py index 3f956b0614c..ec6de4e206d 100644 --- a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py +++ b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py @@ -85,6 +85,11 @@ def message_field(message: object, key: str) -> object: return mapping.get(key) if mapping is not None else getattr(message, key, None) +def string_field(message: object, key: str) -> str | None: + value: Final = message_field(message, key) + return value if isinstance(value, str) else None + + def is_system_message(message: object) -> bool: return message_field(message, "role") == "system" diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index f03004cec15..f198d76f72a 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2,7 +2,9 @@ import copy import json import re import time -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from functools import partial +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast @@ -27,7 +29,10 @@ from litellm.constants import ( from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason from litellm.litellm_core_utils.prompt_templates.common_utils import ( DEFS_MAX_INLINED_BYTES, + invalid_type_message, is_object_schema, + is_present_non_string, + is_unhashable_value_behind, sanitize_input_schema_for_anthropic, schema_fragment_ref, schema_grammar_branch_count, @@ -45,6 +50,8 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( inline_remote_image_urls, ) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( + message_field, + parts_of, place_mid_conversation_system, split_leading_system_run, ) @@ -179,6 +186,76 @@ NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS: Final = 24 NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES: Final = 2048 +def _tool_call_ids_failing( + index: int, tool_calls: object, fails: Callable[[object], bool] +) -> Iterator[tuple[str, object]]: + ids: Final = ((position, message_field(tool_call, "id")) for position, tool_call in enumerate(parts_of(tool_calls))) + return ( + (f"messages[{index}].tool_calls[{position}].id", tool_call_id) + for position, tool_call_id in ids + if fails(tool_call_id) + ) + + +def _roles_failing(messages: Sequence[object], fails: Callable[[object], bool]) -> Iterator[tuple[str, object]]: + roles: Final = ((index, message_field(message, "role")) for index, message in enumerate(messages)) + return ((f"messages[{index}].role", role) for index, role in roles if fails(role)) + + +def _is_non_empty_non_string(value: object) -> bool: + return bool(value) and is_present_non_string(value) + + +def _assistant_tool_call_ids_failing( + messages: Sequence[object], fails: Callable[[object], bool] +) -> Iterator[tuple[str, object]]: + assistant_turns: Final = ( + (index, message) for index, message in enumerate(messages) if message_field(message, "role") == "assistant" + ) + return chain.from_iterable( + _tool_call_ids_failing(index, message_field(message, "tool_calls"), fails) for index, message in assistant_turns + ) + + +def _tool_answer_ids_failing( + messages: Sequence[object], fails: Callable[[object], bool] +) -> Iterator[tuple[str, object]]: + answers: Final = ( + (index, message_field(message, "tool_call_id")) + for index, message in enumerate(messages) + if message_field(message, "role") in ("tool", "function") + ) + return ((f"messages[{index}].tool_call_id", tool_call_id) for index, tool_call_id in answers if fails(tool_call_id)) + + +def _is_non_string_value_behind(error: AttributeError, value: object) -> bool: + return is_present_non_string(value) and value is error.obj + + +def _fields_behind(messages: Sequence[object], error: Exception) -> Iterable[tuple[str, object]]: + match error: + case TypeError(): + caused_error: Final = partial(is_unhashable_value_behind, error) + return chain( + _assistant_tool_call_ids_failing(messages, caused_error), + _tool_answer_ids_failing(messages, caused_error), + _roles_failing(messages, caused_error), + ) + case AttributeError(): + return _assistant_tool_call_ids_failing(messages, partial(_is_non_string_value_behind, error)) + case _: + return () + + +def non_string_message_field_error(messages: Sequence[object], error: Exception) -> AnthropicError | None: + malformed_field: Final = next( + chain(_roles_failing(messages, _is_non_empty_non_string), _fields_behind(messages, error)), None + ) + if malformed_field is None: + return None + return AnthropicError(status_code=400, message=invalid_type_message(*malformed_field)) + + def _tool_type(tool: object) -> str | None: match tool: case {"type": str() as tool_type}: @@ -2370,12 +2447,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Handling anthropic API Prompt Caching if len(anthropic_system_message_list) > 0: optional_params["system"] = anthropic_system_message_list - conversation: Final = place_mid_conversation_system( - later_messages, - supports_mid_conversation_system=supports_mid_conversation_system( - model=model, custom_llm_provider=self.custom_llm_provider - ), - ) + try: + conversation: Final = place_mid_conversation_system( + later_messages, + supports_mid_conversation_system=supports_mid_conversation_system( + model=model, custom_llm_provider=self.custom_llm_provider + ), + ) + except TypeError as error: + field_error: Final = non_string_message_field_error(messages, error) + if field_error is None: + raise + raise field_error from error # Format rest of message according to anthropic guidelines try: anthropic_messages = anthropic_messages_pt( @@ -2385,7 +2468,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): drop_params=drop_params, ) except Exception as e: - raise AnthropicError( + conversion_field_error: Final = non_string_message_field_error(messages, e) + raise conversion_field_error or AnthropicError( status_code=400, message=f"{e}\nReceived Messages={request_messages}", ) # don't use verbose_logger.exception, if exception is raised diff --git a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py index fabc112d443..f1d50fb75e2 100644 --- a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py @@ -74,6 +74,7 @@ class _ClearToolUsesFields(BaseModel): clear_tool_inputs: bool | tuple[str, ...] | None = None +_BLOCK: Final = TypeAdapter(Mapping[str, object]) _BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) # api.anthropic.com, measured 2026-10-19 and 2026-10-28: its wording for the shapes pydantic words differently _API_MESSAGES: Final = MappingProxyType( @@ -317,12 +318,10 @@ def _clear_tool_results( new_blocks: list[object] = [] mutated = False for block in content: - if ( - isinstance(block, dict) - and block.get("type") == "tool_result" - and block.get("tool_use_id") in ids_to_clear - ): - new_blocks.append(_cleared_result(block)) + if (parsed := _parse(_BLOCK, block)) is not None and _block_id( + parsed, "tool_result", "tool_use_id" + ) in ids_to_clear: + new_blocks.append(_cleared_result(parsed)) mutated = True cleared += 1 else: diff --git a/litellm/llms/base_llm/files/batch_records.py b/litellm/llms/base_llm/files/batch_records.py index 070cf73cd57..87ac4bd2b93 100644 --- a/litellm/llms/base_llm/files/batch_records.py +++ b/litellm/llms/base_llm/files/batch_records.py @@ -19,6 +19,8 @@ def responses_batch_body_to_chat_body( openai_request_body: Mapping[str, object], custom_llm_provider: str | None = None, drop_params: bool | None = None, + *, + caller_input: str | ResponseInputParam | None = None, ) -> dict[str, object]: # mutable-ok: provider transforms take the bridged chat body as a plain dict """ Rewrite the body of an OpenAI `/v1/responses` batch record as a Chat Completions body. @@ -52,5 +54,6 @@ def responses_batch_body_to_chat_body( responses_api_request=responses_request, custom_llm_provider=custom_llm_provider, drop_params=drop_params, + caller_input=caller_input, metadata=openai_request_body.get("metadata"), ) diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 0624c2c65db..dcdd9e39c20 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -42,6 +42,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_file_data, text_completion_prompt_to_messages, ) +from litellm.llms.anthropic.common_utils import AnthropicError from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body @@ -347,7 +348,14 @@ class _BatchInputLine: record: _OpenAIBatchRecord -_BATCH_RECORD_CONVERSION_ERRORS: Final = (AttributeError, LookupError, TypeError, ValueError, BadRequestError) +_BATCH_RECORD_CONVERSION_ERRORS: Final = ( + AttributeError, + LookupError, + TypeError, + ValueError, + BadRequestError, + AnthropicError, +) def _batch_line_error(line_number: int, problem: str) -> BedrockError: diff --git a/litellm/responses/litellm_completion_transformation/count_request.py b/litellm/responses/litellm_completion_transformation/count_request.py index a24a26aac42..0b769326012 100644 --- a/litellm/responses/litellm_completion_transformation/count_request.py +++ b/litellm/responses/litellm_completion_transformation/count_request.py @@ -128,6 +128,15 @@ def _with_hoisted_additional_tools(responses_request: Mapping[str, object], drop return {**responses_request, "input": hoisted.input, "tools": list(hoisted.tools)} +def _caller_input(responses_request: Mapping[str, object]) -> ResponseInputParam | None: + responses_input: Final = responses_request.get("input") + return ( + cast(ResponseInputParam, responses_input) # cast-ok: forwarded as sent, as inference does + if isinstance(responses_input, list) + else None + ) + + def _with_emulated_file_search(responses_request: Mapping[str, object]) -> Mapping[str, object]: tools: Final = responses_request.get("tools") if tools is None: @@ -200,6 +209,7 @@ async def chat_count_request( ), custom_llm_provider, policy.drop_params, + caller_input=_caller_input(responses_request), ), model, custom_llm_provider, diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 04fe027cbcc..802ed6bf9c5 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -4,6 +4,7 @@ Handler for transforming responses api requests to litellm.completion requests from collections.abc import Coroutine, Mapping, Sequence from dataclasses import dataclass +from types import MappingProxyType from typing import Final from pydantic import TypeAdapter @@ -32,6 +33,13 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ModelResponse _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object]) +_BRIDGE_TRANSFORM_KEYWORDS: Final = frozenset({"caller_input", "names_non_string_fields"}) + + +def _without_bridge_transform_keywords(request_kwargs: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType( + {key: value for key, value in request_kwargs.items() if key not in _BRIDGE_TRANSFORM_KEYWORDS} + ) def _chat_verbosity(responses_api_request: Mapping[str, object]) -> Mapping[str, object]: @@ -95,6 +103,7 @@ class LiteLLMCompletionTransformationHandler: previous_response_id, hoisted, model=model, + caller_input=input, responses_api_request=responses_api_request, custom_llm_provider=custom_llm_provider, stream=stream, @@ -108,11 +117,13 @@ class LiteLLMCompletionTransformationHandler: **LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model=model, input=bridged_input, + caller_input=input, + names_non_string_fields=True, responses_api_request=bridged_request, custom_llm_provider=custom_llm_provider, stream=stream, extra_headers=extra_headers, - **kwargs, + **_without_bridge_transform_keywords(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)), ), **_chat_verbosity(bridged_request), } @@ -163,6 +174,7 @@ class LiteLLMCompletionTransformationHandler: /, *, model: str, + caller_input: str | ResponseInputParam, responses_api_request: ResponsesAPIOptionalRequestParams, custom_llm_provider: str | None, stream: bool | None, @@ -186,11 +198,13 @@ class LiteLLMCompletionTransformationHandler: **LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model=model, input=bridged.input, + caller_input=caller_input, + names_non_string_fields=True, responses_api_request=bridged.responses_api_request, custom_llm_provider=custom_llm_provider, stream=stream, extra_headers=extra_headers, - **request_kwargs, + **_without_bridge_transform_keywords(request_kwargs), ), **_chat_verbosity(bridged.responses_api_request), }, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 17ae9f2e656..0f4ed0611be 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -45,6 +45,7 @@ from litellm.constants import ( REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER, ) +from litellm.exceptions import BadRequestError from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, block_with_cache_control, @@ -65,6 +66,12 @@ from litellm.litellm_core_utils.hidden_params import ( ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + invalid_type_message, + is_present_non_string, + is_unhashable_value_behind, +) +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control from litellm.llms.anthropic.messages.native.utils import anthropic_stop_reason, openai_chat_refusal_text from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM @@ -268,6 +275,7 @@ _IMAGE_URL_ADAPTER: Final = TypeAdapter(ChatCompletionImageUrlObject) _FILE_PART_ADAPTER: Final = TypeAdapter(_ResponsesFilePart) _MessageT = TypeVar("_MessageT") _MediaT: Final = TypeVar("_MediaT") +_InputItemT: Final = TypeVar("_InputItemT") _CACHED_CONTENT_ADAPTER: Final = TypeAdapter(ChatCompletionCachedContent) _DICT_ITEMS_LIST_ADAPTER: Final = TypeAdapter(list[dict[object, object]]) _TEXT_ADAPTER: Final = TypeAdapter(str) @@ -445,6 +453,86 @@ TOOL_CALL_OUTPUT_ITEM_TYPES: Final = frozenset( ) +def _read_positions( + input_items: Sequence[_InputItemT], read_input: str | ResponseInputParam +) -> Iterator[tuple[int, _InputItemT]]: + read_ids: Final = frozenset(map(id, parts_of(read_input))) + return ((index, item) for index, item in enumerate(input_items) if id(item) in read_ids) + + +def _first_invalid_input_type(fields: Iterator[tuple[str, object]]) -> BadRequestError | None: + first: Final = next(fields, None) + return None if first is None else _invalid_input_type(*first) + + +def _is_tool_output_item(item: object) -> bool: + item_type: Final = message_field(item, "type") + return isinstance(item_type, str) and item_type in TOOL_CALL_OUTPUT_ITEM_TYPES + + +def _converts_before( + convert_input: Callable[[str | ResponseInputParam], object], sent_input: str | ResponseInputParam, position: int +) -> bool: + try: + convert_input(sent_input[:position]) + except TypeError: + return False + return True + + +def _unhashable_tool_output_call_id_error( + input: str | ResponseInputParam, + read_input: str | ResponseInputParam, + error: TypeError, + converts_before: Callable[[int], bool], +) -> BadRequestError | None: + call_ids: Final = ( + (f"input[{index}].call_id", message_field(item, "call_id"), item) + for index, item in _read_positions(parts_of(input), read_input) + if _is_tool_output_item(item) + ) + first: Final = next((candidate for candidate in call_ids if is_unhashable_value_behind(error, candidate[1])), None) + if first is None: + return None + path, call_id, item = first + read_position: Final = next( + position for position, read_item in enumerate(parts_of(read_input)) if read_item is item + ) + return _invalid_input_type(path, call_id) if converts_before(read_position) else None + + +def _carries_non_string_field(message: object) -> bool: + tool_call_ids: Final = ( + message_field(tool_call, "id") for tool_call in parts_of(message_field(message, "tool_calls")) + ) + return is_present_non_string(message_field(message, "role")) or any(map(is_present_non_string, tool_call_ids)) + + +def _non_string_item_fields( + item: Mapping[str, object], convert_item: Callable[[Mapping[str, object]], Sequence[object]] +) -> Iterator[tuple[str, object]]: + messages: Final = convert_item(item) + tool_call_id_field: Final = "call_id" if item.get("call_id") else "id" + roles: Final = (("role", message_field(message, "role")) for message in messages) + tool_calls: Final = chain.from_iterable(parts_of(message_field(message, "tool_calls")) for message in messages) + tool_call_ids: Final = ((tool_call_id_field, message_field(tool_call, "id")) for tool_call in tool_calls) + return ((field, value) for field, value in chain(roles, tool_call_ids) if is_present_non_string(value)) + + +def _indexed_non_string_item_fields( + input: ResponseInputParam, + read_input: str | ResponseInputParam, + convert_item: Callable[[Mapping[str, object]], Sequence[object]], +) -> Iterator[tuple[str, object]]: + for index, item in _read_positions(input, read_input): + for field, value in _non_string_item_fields(item, convert_item): + yield f"input[{index}].{field}", value + + +def _invalid_input_type(field_path: str, value: object) -> BadRequestError: + return BadRequestError(message=invalid_type_message(field_path, value), model="", llm_provider="") + + def _input_item_role(input_item: Mapping[str, object]) -> str: return cast(str, input_item.get("role") or "user") # cast-ok: client-supplied role forwarded verbatim, unvalidated @@ -1101,6 +1189,9 @@ class LiteLLMCompletionResponsesConfig: stream: bool | None = None, extra_headers: Mapping[str, object] | None = None, drop_params: object = None, + *, + caller_input: str | ResponseInputParam | None = None, + names_non_string_fields: bool = False, **kwargs, ) -> dict: """ @@ -1149,6 +1240,7 @@ class LiteLLMCompletionResponsesConfig: native_compaction: Final = _replays_compaction_block( request_input, context_management ) and provider_takes_compaction_blocks(model, custom_llm_provider) + numbered_input: Final = request_input if caller_input is None else caller_input bridged_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=request_input, responses_api_request=responses_api_request, @@ -1157,7 +1249,28 @@ class LiteLLMCompletionResponsesConfig: tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs), native_compaction=native_compaction, only_since_compaction=not native_compaction, + caller_input=numbered_input, ) + malformed_field_error: Final = ( + _first_invalid_input_type( + _indexed_non_string_item_fields( + numbered_input, + _input_to_read(request_input, not native_compaction, native_compaction), + partial( + LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message, + replay_reasoning=True, + native_compaction=native_compaction, + ), + ) + ) + if names_non_string_fields + and isinstance(numbered_input, list) + and any(map(_carries_non_string_field, bridged_messages)) + and provider_builds_anthropic_request(model, custom_llm_provider) + else None + ) + if malformed_field_error is not None: + raise malformed_field_error litellm_completion_request: dict = { "messages": ( @@ -1221,6 +1334,7 @@ class LiteLLMCompletionResponsesConfig: tool_call_provider_fields: bool = False, native_compaction: bool = False, only_since_compaction: bool = False, + caller_input: str | ResponseInputParam | None = None, ) -> list[ AllMessageValues | GenericChatCompletionMessage @@ -1274,13 +1388,27 @@ class LiteLLMCompletionResponsesConfig: dumped_input: Final = with_compaction_items_as_params(input) read_input: Final = _input_to_read(dumped_input, only_since_compaction, native_compaction) - messages.extend( - LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( - input=_without_item_keys(read_input, _dropped_item_keys(item_cache_control, tool_call_provider_fields)), - replay_reasoning=replay_reasoning, - native_compaction=native_compaction, - ) + sent_input: Final = _without_item_keys( + read_input, _dropped_item_keys(item_cache_control, tool_call_provider_fields) ) + convert_input: Final = partial( + LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message, + replay_reasoning=replay_reasoning, + native_compaction=native_compaction, + ) + try: + converted: Final = convert_input(sent_input) + except TypeError as error: + call_id_error: Final = _unhashable_tool_output_call_id_error( + dumped_input if caller_input is None else caller_input, + read_input, + error, + partial(_converts_before, convert_input, sent_input), + ) + if call_id_error is None: + raise + raise call_id_error from error + messages.extend(converted) return messages diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 05a63030c69..778c8dfe2f5 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -5042,3 +5042,50 @@ def test_cache_control_hook_injects_next_to_a_malformed_message_instead_of_raisi {"type": "ephemeral"}, {"type": "ephemeral"}, ] + + +def _search_conversation_with(follow_up: dict[str, object]) -> list[AllMessageValues]: + return [ + cast(AllMessageValues, {"role": "user", "content": "Weather in Paris?"}), + _replayed_search_call(_SEARCH_CALL), + cast(AllMessageValues, follow_up), + cast(AllMessageValues, _GO_ON), + ] + + +def _cache_marks_after_injection(messages: list[AllMessageValues]) -> tuple[int, tuple[int, ...]]: + _, processed, _ = AnthropicCacheControlHook().get_chat_completion_prompt( + model="anthropic/claude-opus-5-5", + messages=messages, + non_default_params={"cache_control_injection_points": [{"location": "message", "role": "user"}]}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return ( + AnthropicCacheControlHook.count_request_cache_breakpoints(messages), + tuple(_marks_in(message) for message in processed), + ) + + +@pytest.mark.parametrize( + ("malformed", "well_formed"), + [ + pytest.param( + {**_SEARCH_ANSWER, "tool_call_id": {"id": "srvtoolu_01Search"}}, + {**_SEARCH_ANSWER, "tool_call_id": "toolu_01Unrelated"}, + id="object_tool_call_id", + ), + pytest.param( + {**_SEARCH_ANSWER, "tool_call_id": ["srvtoolu_01Search"]}, + {**_SEARCH_ANSWER, "tool_call_id": "toolu_01Unrelated"}, + id="list_tool_call_id", + ), + ], +) +def test_cache_control_hook_reads_a_non_string_answer_id_as_answering_no_tool_call( + malformed: dict[str, object], well_formed: dict[str, object] +): + assert _cache_marks_after_injection(_search_conversation_with(malformed)) == _cache_marks_after_injection( + _search_conversation_with(well_formed) + ) diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index c8a2f783809..636e23168cf 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -25,7 +25,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt from litellm.llms.anthropic.chat import ModelResponseIterator -from litellm.llms.anthropic.common_utils import process_anthropic_headers +from litellm.llms.anthropic.common_utils import AnthropicError, process_anthropic_headers from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema from litellm.llms.anthropic.messages.native.transformation import ( @@ -9922,6 +9922,382 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key(): assert litellm_params["metadata"] == {} +_KEPT_SEARCH_CALL: Final = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "tool_search_tool_bm25", "arguments": "{}"}, + } + ], + "provider_specific_fields": { + "tool_results": [ + { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_01Search", + "content": {"type": "tool_search_tool_search_result", "tool_references": []}, + } + ] + }, +} +_WEATHER_CALL: Final = { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "toolu_01", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}], +} +_ASK: Final = {"role": "user", "content": "Weather in Paris?"} + + +def _search_answered_with(tool_call_id: object) -> list[dict[str, object]]: + return [ + _ASK, + _KEPT_SEARCH_CALL, + {"role": "tool", "tool_call_id": tool_call_id, "content": "found get_weather"}, + {"role": "user", "content": "Go on"}, + ] + + +def _orphan_answer_with(tool_call_id: object) -> list[dict[str, object]]: + return [ + _ASK, + {"role": "assistant", "content": "Checking"}, + {"role": "tool", "tool_call_id": tool_call_id, "content": "sunny"}, + {"role": "user", "content": "Go on"}, + ] + + +@pytest.mark.parametrize( + ("messages", "field_path", "type_name"), + [ + pytest.param( + [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello"}, + {"role": ["user"], "content": "Go on"}, + {"role": "system", "content": "Answer in one sentence."}, + ], + "messages[2].role", + "list", + id="list_role_before_a_later_system_message", + ), + pytest.param( + [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello"}, + {"role": ["user"], "content": "Go on"}, + ], + "messages[2].role", + "list", + id="list_role", + ), + pytest.param( + [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello"}, + {"role": 123, "content": "Go on"}, + {"role": "system", "content": "Answer in one sentence."}, + ], + "messages[2].role", + "int", + id="int_role_before_a_later_system_message", + ), + pytest.param( + [*_orphan_answer_with({"id": "toolu_01"})[:3], {"role": True, "content": "Go on"}], + "messages[3].role", + "bool", + id="bool_role_after_an_orphan_object_tool_call_id", + ), + pytest.param( + _search_answered_with({"id": "srvtoolu_01Search"}), + "messages[2].tool_call_id", + "dict", + id="object_tool_call_id_after_a_kept_server_result", + ), + pytest.param( + _search_answered_with(["srvtoolu_01Search"]), + "messages[2].tool_call_id", + "list", + id="list_tool_call_id_after_a_kept_server_result", + ), + pytest.param( + [_ASK, _WEATHER_CALL, {"role": "tool", "tool_call_id": {"id": "toolu_01"}, "content": "sunny"}], + "messages[2].tool_call_id", + "dict", + id="object_tool_call_id_answering_a_client_tool_call", + ), + pytest.param( + [ + _ASK, + _WEATHER_CALL, + {"role": "function", "name": "get_weather", "tool_call_id": {"id": "toolu_01"}, "content": "sunny"}, + ], + "messages[2].tool_call_id", + "dict", + id="object_tool_call_id_on_a_function_message", + ), + pytest.param( + [ + _ASK, + {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]}, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ], + "messages[1].tool_calls[0].id", + "dict", + id="object_assistant_tool_call_id", + ), + pytest.param( + [ + _ASK, + {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": 7}]}, + {"role": "tool", "tool_call_id": "7", "content": "sunny"}, + ], + "messages[1].tool_calls[0].id", + "int", + id="int_assistant_tool_call_id", + ), + pytest.param( + [*_orphan_answer_with({"id": "toolu_01"})[:3], {"role": ["user"], "content": "Go on"}], + "messages[3].role", + "list", + id="list_role_after_an_orphan_object_tool_call_id", + ), + pytest.param( + [ + *_orphan_answer_with({"id": "toolu_01"})[:3], + {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]}, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ], + "messages[3].tool_calls[0].id", + "dict", + id="object_assistant_tool_call_id_after_an_orphan_object_tool_call_id", + ), + pytest.param( + [ + {**_ASK, "tool_call_id": {"id": "stray"}}, + _WEATHER_CALL, + {"role": "tool", "tool_call_id": {"id": "toolu_01"}, "content": "sunny"}, + ], + "messages[2].tool_call_id", + "dict", + id="object_tool_call_id_after_a_stray_id_on_a_user_message", + ), + pytest.param( + [ + {**_ASK, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "stray"}}]}, + {**_WEATHER_CALL, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]}, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ], + "messages[1].tool_calls[0].id", + "dict", + id="object_assistant_tool_call_id_after_stray_tool_calls_on_a_user_message", + ), + pytest.param( + [_ASK, {"role": [], "content": "ok"}, {"role": "system", "content": "Be terse."}], + "messages[1].role", + "list", + id="empty_list_role_before_a_system_message", + ), + pytest.param( + [_ASK, {"role": {}, "content": "ok"}, {"role": "user", "content": "again"}], + "messages[1].role", + "dict", + id="empty_object_role", + ), + ], +) +@pytest.mark.parametrize("modify_params", [False, True]) +@pytest.mark.parametrize("model", [UNFLAGGED_CLAUDE, FLAGGED_CLAUDE]) +def test_chat_transform_request_answers_400_naming_a_non_string_role_or_tool_call_id( + local_model_cost_map, + monkeypatch: pytest.MonkeyPatch, + messages: list[dict[str, object]], + field_path: str, + type_name: str, + modify_params: bool, + model: str, +): + monkeypatch.setattr(litellm, "modify_params", modify_params) + + with pytest.raises(AnthropicError) as raised: + _chat_request(AnthropicConfig(), model, copy.deepcopy(messages)) + + assert (raised.value.status_code, raised.value.message) == ( + 400, + f"Invalid type for '{field_path}': expected a string, got {type_name}", + ) + + +@pytest.mark.parametrize(("role", "type_name"), [(["user"], "list"), ({"name": "user"}, "dict"), ([], "list")]) +def test_chat_transform_request_names_a_non_string_role_on_a_cache_marked_message_under_drop_params( + local_model_cost_map, role: object, type_name: str +): + marked: Final = {"role": role, "content": [{"type": "text", "text": "ok"}], "cache_control": {"type": "ephemeral"}} + + with pytest.raises(AnthropicError) as raised: + AnthropicConfig().transform_request( + model=UNFLAGGED_CLAUDE, + messages=[_ASK, marked], + optional_params={}, + litellm_params={"drop_params": True}, + headers={}, + ) + + assert (raised.value.status_code, raised.value.message) == ( + 400, + f"Invalid type for 'messages[1].role': expected a string, got {type_name}", + ) + + +@pytest.mark.parametrize( + ("malformed", "well_formed", "modify_params"), + [ + pytest.param( + [{**_ASK, "tool_call_id": {"id": "stray"}}], [_ASK], False, id="stray_tool_call_id_on_a_user_message" + ), + pytest.param( + [{**_ASK, "tool_calls": [{**_WEATHER_CALL["tool_calls"][0], "id": {"value": "toolu_01"}}]}], + [_ASK], + False, + id="stray_tool_calls_on_a_user_message", + ), + pytest.param( + _orphan_answer_with({"id": "toolu_01"}), + _orphan_answer_with("toolu_01"), + True, + id="orphan_answer_dropped_under_modify_params", + ), + ], +) +def test_chat_transform_request_keeps_converting_a_non_string_id_it_never_reads( + local_model_cost_map, + monkeypatch: pytest.MonkeyPatch, + malformed: list[dict[str, object]], + well_formed: list[dict[str, object]], + modify_params: bool, +): + monkeypatch.setattr(litellm, "modify_params", modify_params) + + converted: Final = _chat_request(AnthropicConfig(), UNFLAGGED_CLAUDE, copy.deepcopy(malformed)) + + assert ( + converted["messages"] + == _chat_request(AnthropicConfig(), UNFLAGGED_CLAUDE, copy.deepcopy(well_formed))["messages"] + ) + + +def test_chat_transform_request_keeps_a_conversion_error_that_a_dropped_orphan_answer_did_not_cause( + local_model_cost_map, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "modify_params", True) + + with pytest.raises(AnthropicError) as raised: + _chat_request( + AnthropicConfig(), + UNFLAGGED_CLAUDE, + [*_orphan_answer_with({"id": "toolu_01"})[:3], {"role": "bogus", "content": "Go on"}], + ) + + assert raised.value.status_code == 400 + assert "Invalid Message passed in {'role': 'bogus'" in raised.value.message + + +@pytest.mark.parametrize( + ("later_turn", "real_cause"), + [ + pytest.param( + [{"role": "user", "content": ["hi", "there"]}], + "'str' object has no attribute 'get'", + id="string_content_parts", + ), + pytest.param( + [ + { + **_WEATHER_CALL, + "tool_calls": [ + { + **_WEATHER_CALL["tool_calls"][0], + "function": {"name": "get_weather", "arguments": {"city": "Paris"}}, + } + ], + }, + {"role": "tool", "tool_call_id": "toolu_01", "content": "rain"}, + ], + "'dict' object has no attribute 'strip'", + id="object_tool_call_arguments", + ), + pytest.param( + [{"role": "user", "content": [{"type": "image_url", "image_url": ["x"]}]}], + "list indices must be integers or slices, not str", + id="list_image_url", + ), + ], +) +@pytest.mark.parametrize("dropped_answer_id", [7, {"id": "toolu_01"}], ids=["int", "object"]) +def test_chat_transform_request_never_blames_a_dropped_tool_answer_id_for_another_failure( + local_model_cost_map, + monkeypatch: pytest.MonkeyPatch, + later_turn: list[dict[str, object]], + real_cause: str, + dropped_answer_id: object, +): + monkeypatch.setattr(litellm, "modify_params", True) + + with pytest.raises(AnthropicError) as raised: + _chat_request(AnthropicConfig(), UNFLAGGED_CLAUDE, [*_orphan_answer_with(dropped_answer_id)[:3], *later_turn]) + + assert raised.value.status_code == 400 + assert raised.value.message.startswith(real_cause) + + +def test_chat_transform_request_never_blames_a_numeric_assistant_tool_call_id_for_an_earlier_failure( + local_model_cost_map, +): + with pytest.raises(AnthropicError) as raised: + _chat_request( + AnthropicConfig(), + UNFLAGGED_CLAUDE, + [ + {"role": "user", "content": ["hi", "there"]}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": 5, "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "5", "content": "rain"}, + ], + ) + + assert raised.value.status_code == 400 + assert raised.value.message.startswith("'str' object has no attribute 'get'") + + +@pytest.mark.parametrize("empty_role", [0, False]) +def test_chat_transform_request_never_blames_an_empty_role_for_another_failure(local_model_cost_map, empty_role): + list_part_type_after_an_empty_role: Final = [ + _ASK, + {"role": empty_role, "content": "ok"}, + {"role": "user", "content": [{"type": ["text"], "text": "again"}]}, + {"role": "system", "content": "Be terse."}, + ] + + with pytest.raises(TypeError): + _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, list_part_type_after_an_empty_role) + + +def test_chat_transform_request_names_no_field_for_a_failure_outside_roles_and_tool_call_ids(local_model_cost_map): + list_part_type: Final = [ + _ASK, + {"role": "assistant", "content": "ok"}, + {"role": "user", "content": [{"type": ["text"], "text": "again"}]}, + {"role": "system", "content": "Be terse."}, + ] + + with pytest.raises(TypeError): + _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, list_part_type) + + def _five_cache_hint_conversation() -> list[dict[str, object]]: return [ {"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]}, diff --git a/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py index 4cbf4cfa5f6..70eddd9e16e 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py @@ -614,9 +614,18 @@ def test_clear_tool_inputs_list_clears_only_the_named_tools(): } +def _history_with_first_block_field(index: int, field: str, value: object) -> list[dict[str, object]]: + history: Final = _make_history(n_pairs=4) + return [ + {**message, "content": [{**message["content"][0], field: value}, *message["content"][1:]]} + if position == index + else message + for position, message in enumerate(history) + ] + + def test_a_tool_use_with_a_non_string_id_is_left_for_the_provider_to_reject(): - messages = _make_history(n_pairs=4) - messages[1]["content"][0]["id"] = ["toolu_00"] + messages: Final = _history_with_first_block_field(1, "id", ["toolu_00"]) new_messages, applied = apply_clear_tool_uses_20250919( model=MODEL, messages=messages, @@ -635,6 +644,24 @@ def test_a_tool_use_with_a_non_string_id_is_left_for_the_provider_to_reject(): assert (new_messages[3]["content"][0]["input"], new_messages[5]["content"][0]["input"]) == ({}, {}) +def test_a_tool_result_with_a_non_string_tool_use_id_is_left_for_the_provider_to_reject(): + messages: Final = _history_with_first_block_field(2, "tool_use_id", ["toolu_00"]) + new_messages, applied = apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + }, + ) + assert applied is not None + assert new_messages[2] == messages[2] + assert _cleared_ids(new_messages[3:]) == {"toolu_01", "toolu_02"} + + async def _malformed_spec_error(edits: list[dict[str, object]]) -> AnthropicContextManagementError: with pytest.raises(AnthropicContextManagementError) as raised: await apply_context_management( diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 9d9e39df944..512e1f325f4 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -4472,6 +4472,17 @@ def test_file_content_conversion_keeps_a_line_whose_json_holds_unicode_line_sepa assert bedrock_record["modelInput"]["messages"] == [{"role": "user", "content": [{"text": text}]}] +_CLAUDE_BODY_WITH_A_LIST_ROLE_BEFORE_A_SYSTEM_MESSAGE: Final = { + "model": "bedrock/us.anthropic.claude-haiku-5-5", + "max_tokens": 20, + "messages": [ + {"role": "user", "content": "hi"}, + {"role": ["user"], "content": "ok"}, + {"role": "system", "content": "Be terse."}, + ], +} + + def _batch_line(body: object) -> str: return json.dumps({**_VALID_BATCH_RECORD, "body": body}) @@ -4516,6 +4527,11 @@ def _batch_line_with_body(field: str, value: object) -> str: "Batch input file line 2 could not be converted to a Bedrock batch record", id="string-messages", ), + pytest.param( + f"{_VALID_BATCH_LINE}\n{_batch_line(_CLAUDE_BODY_WITH_A_LIST_ROLE_BEFORE_A_SYSTEM_MESSAGE)}\n".encode(), + "Batch input file line 2 could not be converted to a Bedrock batch record", + id="claude-list-role-before-a-system-message", + ), pytest.param( f"{_VALID_BATCH_LINE}\n{'[' * 1_000_000}{']' * 1_000_000}\n".encode(), "Batch input file line 2 could not be parsed", diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index bc73ef8598a..5ae985ea7e5 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -1598,6 +1598,53 @@ async def test_previous_response_id_keeps_the_tools_an_earlier_codex_tool_search assert prisma_client.db.queries == 1 +@pytest.mark.asyncio +async def test_a_codex_additional_tools_item_sends_claude_its_tools_but_not_its_text() -> None: + anthropic: Final = _RecordingRawAnthropicMessages() + + await litellm.aresponses( + model=_MODEL, + input=[ + {"type": "additional_tools", "role": "developer", "content": "Reply PINEAPPLE.", "tools": [_SHELL_TOOL]}, + {"role": "user", "content": "Hi"}, + ], + api_key="sk-ant-fake", + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + ) + + sent: Final = json.loads(anthropic.body) + assert [tool["name"] for tool in sent["tools"]] == ["shell"] + assert "PINEAPPLE" not in json.dumps(sent) + + +@pytest.mark.asyncio +async def test_previous_response_id_names_a_malformed_item_after_a_codex_additional_tools_item_by_its_index() -> None: + weather_tool: Final = { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + } + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_SEARCH_TURN, _CALENDAR_TURN])), + pytest.raises(litellm.BadRequestError) as raised, + ): + await litellm.aresponses( + model=_MODEL, + previous_response_id="chatcmpl-calendar-turn", + input=[ + {"type": "additional_tools", "role": "developer", "tools": [weather_tool]}, + {"type": "function_call_output", "call_id": "toolu_standup", "output": "Created Standup."}, + {"role": ["user"], "content": "Now create another event titled Retro."}, + ], + tools=[_CODEX_TOOL_SEARCH, _SHELL_TOOL], + api_key="sk-ant-fake", + client=AsyncHTTPHandler(transport=httpx.MockTransport(_RecordingRawAnthropicMessages())), + ) + + assert "'input[2].role'" in raised.value.message + + _REFUSAL_EXPLANATION: Final = ANTHROPIC_DOCS_REFUSAL_MESSAGE["stop_details"]["explanation"] _ANSWER_BEFORE_REFUSAL: Final = {"type": "text", "text": "Here is the start."} @@ -1808,7 +1855,11 @@ _RAN_SEARCH_RESULT: Final = { {"type": "web_search_result", "url": "https://tokyo.example", "title": "Tokyo", "encrypted_content": "e"} ], } -_THINKING_BEFORE_SEARCHES: Final = {"type": "thinking", "thinking": "Find each sister city in turn.", "signature": "sig"} +_THINKING_BEFORE_SEARCHES: Final = { + "type": "thinking", + "thinking": "Find each sister city in turn.", + "signature": "sig", +} _TEXT_BEFORE_PENDING_SEARCH: Final = {"type": "text", "text": "Tokyo pairs with Paris. Checking Paris next."} _PENDING_SEARCH: Final = { "type": "server_tool_use", diff --git a/tests/unit/responses/litellm_completion_transformation/test_count_request.py b/tests/unit/responses/litellm_completion_transformation/test_count_request.py index 2080a091160..e0a5997eac7 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_count_request.py +++ b/tests/unit/responses/litellm_completion_transformation/test_count_request.py @@ -86,7 +86,13 @@ async def test_claude_count_request_counts_additional_tools_input_items_as_tools policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset()) hoisted: Final = await _claude_count( - {"input": [{"type": "additional_tools", "tools": [_WEATHER_TOOL]}, _QUESTION]}, policy + { + "input": [ + {"type": "additional_tools", "role": "developer", "content": "Reply PINEAPPLE.", "tools": [_WEATHER_TOOL]}, + _QUESTION, + ] + }, + policy, ) assert isinstance(hoisted, ChatCountRequest) @@ -129,6 +135,17 @@ async def test_claude_count_request_rejects_a_malformed_tool_flag_without_drop_p assert "defer_loading" in flagged.message +@pytest.mark.asyncio +async def test_claude_count_request_leaves_a_non_string_role_for_the_count_endpoint_to_judge() -> None: + counted: Final = await _claude_count( + {"input": [{"type": "additional_tools", "tools": [_WEATHER_TOOL]}, _QUESTION, {"role": ["user"], "content": "?"}]}, + DropPolicy(drop_params=False, additional_drop_params=frozenset()), + ) + + assert isinstance(counted, ChatCountRequest) + assert [message.get("role") for message in counted.messages] == ["user", ["user"]] + + @pytest.mark.asyncio async def test_claude_count_request_leaves_out_tools_the_deployment_drops() -> None: policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset({"tools"})) diff --git a/tests/unit/responses/litellm_completion_transformation/test_handler.py b/tests/unit/responses/litellm_completion_transformation/test_handler.py index d731bf5f41c..4794f3701c5 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_handler.py @@ -246,6 +246,26 @@ async def test_bridged_turn_with_a_client_field_named_previous_response_client_i assert isinstance(response, ResponsesAPIResponse) +@pytest.mark.asyncio +@pytest.mark.parametrize("client_field", ["caller_input", "names_non_string_fields"]) +async def test_bridged_turn_with_a_client_field_named_like_a_bridge_keyword_reaches_the_provider(client_field: str): + provider: Final = _RecordingAnthropicHandler(_ANTHROPIC_MESSAGE_PAYLOAD) + client: Final = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider)) + + response = await litellm.aresponses( + model="azure_ai/claude-sonnet-4-6", + api_base="https://fake-foundry-resource.services.ai.azure.com", + api_key="fake-api-key", + input="Double it", + client=client, + **{client_field: "from the client body"}, + ) + + assert provider.request_body is not None, "the bridged turn never reached the provider" + assert isinstance(response, ResponsesAPIResponse) + + class _StoredTurns: def __init__(self, rows: tuple[Mapping[str, object], ...]) -> None: self.rows: Final = rows @@ -310,3 +330,32 @@ async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_h ("assistant", json.dumps([{"type": "text", "text": "Noted."}])), ("user", json.dumps([{"type": "text", "text": "Double it"}])), ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("client_field", ["caller_input", "names_non_string_fields"]) +async def test_bridged_follow_up_after_a_client_tool_search_with_a_client_field_named_like_a_bridge_keyword_reaches_the_provider( + client_field: str, +): + previous_response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="azure_ai", model_id="deployment-a", response_id="chatcmpl-turn-one" + ) + stored_turns: Final = _StoredTurns((_stored_mcp_turn(previous_response_id),)) + provider: Final = _RecordingAnthropicHandler(_ANTHROPIC_MESSAGE_PAYLOAD) + client: Final = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider)) + + with patch("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=stored_turns)): + response = await litellm.aresponses( + model="azure_ai/claude-sonnet-4-6", + api_base="https://fake-foundry-resource.services.ai.azure.com", + api_key="fake-api-key", + input="Double it", + tools=[{"type": "tool_search", "execution": "client"}], + previous_response_id=previous_response_id, + client=client, + **{client_field: "from the client body"}, + ) + + assert provider.request_body is not None, "the bridged turn never reached the provider" + assert isinstance(response, ResponsesAPIResponse) diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index bb1f5104200..b4cd606374a 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -18,6 +18,7 @@ from openai.types.responses.response_function_web_search import ( from pydantic import TypeAdapter, ValidationError import litellm +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field from litellm.constants import CALLER_SESSION_ID_METADATA_KEY from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR @@ -7189,3 +7190,207 @@ async def test_bridge_partial_response_on_disconnect_bills_only_the_input_since_ assert await _input_tokens_billed_on_disconnect( cast(ResponseInputParam, [*history_it_replaces, *since_compaction]) ) == await _input_tokens_billed_on_disconnect(since_compaction) + + +_ANTHROPIC_MODEL: Final = "anthropic/claude-sonnet-5-5" +_WEATHER_ASK: Final = {"role": "user", "content": "Weather in Paris?"} + + +def _weather_call(**fields: object) -> dict[str, object]: + return {"type": "function_call", "name": "get_weather", "arguments": "{}", **fields} + + +def _bridge_to(model: str, history: list[dict[str, object]], **request: object) -> dict: + return LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model=model, + input=cast(ResponseInputParam, history), + responses_api_request=cast(ResponsesAPIOptionalRequestParams, request), + names_non_string_fields=True, + ) + + +@pytest.mark.parametrize("output_type", ["function_call_output", "custom_tool_call_output"]) +@pytest.mark.parametrize(("call_id", "type_name"), [({"id": "call_weather"}, "dict"), (["call_weather"], "list")]) +def test_a_tool_output_with_an_unhashable_call_id_is_a_400_naming_its_input_item( + output_type: str, call_id: object, type_name: str +) -> None: + history: Final = cast( + ResponseInputParam, + [_WEATHER_ASK, _weather_call(call_id="call_weather"), {"type": output_type, "call_id": call_id, "output": "sunny"}], + ) + + with pytest.raises(litellm.BadRequestError) as raised: + LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=history, responses_api_request={} + ) + + assert raised.value.message == ( + f"litellm.BadRequestError: Invalid type for 'input[2].call_id': expected a string, got {type_name}" + ) + + +def test_a_malformed_call_id_after_a_compaction_item_is_named_by_its_index_in_the_request() -> None: + history: Final = cast( + ResponseInputParam, + [ + {"role": "user", "content": "Plan a Tokyo trip."}, + {"role": "assistant", "content": "Day one, Asakusa."}, + {"type": "compaction", "id": "cmp_litellm", "encrypted_content": json.dumps(_COMPACTION_BLOCK)}, + _weather_call(call_id="call_weather"), + {"type": "function_call_output", "call_id": {"id": "call_weather"}, "output": "sunny"}, + ], + ) + + with pytest.raises(litellm.BadRequestError) as raised: + LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=history, responses_api_request={}, only_since_compaction=True + ) + + assert "'input[4].call_id'" in raised.value.message + + +@pytest.mark.parametrize("call_id", [7, {"id": "call_weather"}, ["call_weather"]], ids=["int", "object", "list"]) +def test_a_conversion_failure_elsewhere_is_never_blamed_on_a_call_id(call_id: object) -> None: + history: Final = cast( + ResponseInputParam, + [ + _WEATHER_ASK, + {"type": ["message"], "role": "user", "content": "Go on"}, + {"type": "function_call_output", "call_id": call_id, "output": "sunny"}, + ], + ) + + with pytest.raises((TypeError, litellm.BadRequestError)) as raised: + LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=history, responses_api_request={} + ) + + assert "call_id" not in str(raised.value) + + +def test_a_tool_output_with_an_int_call_id_converts_like_its_string_form() -> None: + def converted(call_id: object) -> list[object]: + return list( + LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=cast( + ResponseInputParam, + [_WEATHER_ASK, {"type": "function_call_output", "call_id": call_id, "output": "sunny"}], + ), + responses_api_request={}, + ) + ) + + assert converted(7) == converted("7") + + +def test_a_non_string_role_still_converts_for_a_target_that_judges_it_itself() -> None: + history: Final = [{"role": "user", "content": "Hi"}, {"role": ["user"], "content": "Go on"}] + + shared: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=cast(ResponseInputParam, history), responses_api_request={} + ) + bridged: Final = _bridge_to("gemini/gemini-3.8-flash", history) + + assert [message_field(message, "role") for message in shared] == ["user", ["user"]] + assert [message_field(message, "role") for message in bridged["messages"]] == ["user", ["user"]] + + +@pytest.mark.parametrize( + ("history", "field_path", "type_name"), + [ + pytest.param( + [{"role": "user", "content": "Hi"}, {"role": ["user"], "content": "Go on"}], + "input[1].role", + "list", + id="list_role", + ), + pytest.param( + [{"role": "user", "content": "Hi"}, {"role": {"name": "user"}, "content": "Go on"}], + "input[1].role", + "dict", + id="dict_role", + ), + pytest.param( + [{"role": "user", "content": "Hi"}, {"role": 7, "content": "Go on"}], "input[1].role", "int", id="int_role" + ), + pytest.param( + [_WEATHER_ASK, _weather_call(call_id={"id": "call_weather"})], + "input[1].call_id", + "dict", + id="dict_function_call_id", + ), + pytest.param( + [_WEATHER_ASK, _weather_call(call_id=7), {"type": "function_call_output", "call_id": "7", "output": "sunny"}], + "input[1].call_id", + "int", + id="int_function_call_id", + ), + pytest.param( + [_WEATHER_ASK, _weather_call(id=["fc_weather"])], "input[1].id", "list", id="list_function_call_item_id" + ), + pytest.param( + [ + {"role": "user", "content": "Plan a Tokyo trip."}, + {"role": "assistant", "content": "Day one, Asakusa."}, + {"type": "compaction", "id": "cmp_litellm", "encrypted_content": json.dumps(_COMPACTION_BLOCK)}, + {"role": ["user"], "content": "And Osaka?"}, + ], + "input[3].role", + "list", + id="list_role_after_a_compaction_item", + ), + ], +) +def test_a_non_string_field_bridged_to_anthropic_is_a_400_naming_its_input_item( + history: list[dict[str, object]], field_path: str, type_name: str +) -> None: + with pytest.raises(litellm.BadRequestError) as raised: + _bridge_to(_ANTHROPIC_MODEL, history, instructions="Be terse.") + + assert raised.value.message == ( + f"litellm.BadRequestError: Invalid type for '{field_path}': expected a string, got {type_name}" + ) + + +def test_a_non_string_role_on_an_item_that_never_reads_it_still_bridges_to_anthropic() -> None: + history: Final = [ + _WEATHER_ASK, + _weather_call(call_id="call_weather", role=["x"]), + {"type": "function_call_output", "call_id": "call_weather", "output": "sunny", "role": ["x"]}, + ] + + bridged: Final = _bridge_to(_ANTHROPIC_MODEL, history) + + assert [message_field(message, "role") for message in bridged["messages"]] == ["user", "assistant", "tool"] + + +@pytest.mark.parametrize( + ("history", "field_path"), + [ + pytest.param([_WEATHER_ASK, {"role": ["user"], "content": "Go on"}], "input[2].role", id="role"), + pytest.param( + [ + _WEATHER_ASK, + _weather_call(call_id="call_weather"), + {"type": "function_call_output", "call_id": {"id": "call_weather"}, "output": "sunny"}, + ], + "input[3].call_id", + id="tool_output_call_id", + ), + ], +) +def test_a_malformed_item_after_a_codex_additional_tools_item_is_named_by_its_index_in_the_request( + history: list[dict[str, object]], field_path: str +) -> None: + tools_item: Final = { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}}], + } + + with pytest.raises(litellm.BadRequestError) as raised: + litellm.responses( + model=_ANTHROPIC_MODEL, api_key="sk-ant-test", input=cast(ResponseInputParam, [tools_item, *history]) + ) + + assert f"'{field_path}'" in raised.value.message