diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 8e6341805db..9902fe1d8fc 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -41,12 +41,14 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import ( ) from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( prompt_cache_breakpoint_with_readable_mode, - responses_reasoning_items_from_thinking_blocks, whole_prompt_cache_breakpoint_or_none, with_prompt_cache_breakpoint, ) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + responses_reasoning_items_from_thinking_blocks, +) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.bridges.completion_transformation import ( CompletionTransformationBridge, diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 4e473c2f3f0..ea518d506fb 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -26,18 +26,18 @@ from litellm.exceptions import UnsupportedParamsError from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.integrations.prompt_management_base import PromptManagementClient -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.factory import ( + tool_call_ids_answered_by_client, + tool_call_ids_answered_in, +) +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import string_field +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT, cache_control_as_prompt_cache_breakpoint, prompt_cache_breakpoint_as_cache_control, with_prompt_cache_breakpoint, without_prompt_cache_breakpoints, ) -from litellm.litellm_core_utils.prompt_templates.factory import ( - tool_call_ids_answered_by_client, - tool_call_ids_answered_in, -) -from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import string_field from litellm.llms.anthropic.common_utils import ( is_claude_code_one_shot_subagent_request, supports_anthropic_cache_control, diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 08ab2d1a2e5..9fb4ad27d75 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -19,7 +19,7 @@ from litellm.constants import ( VLLM_CHOICE_FIELDS, ) from litellm.litellm_core_utils.hidden_params import get_hidden_params -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( extract_reasoning_content, ) from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index f48a42963cf..8f874edfddf 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2,52 +2,212 @@ Common utility functions used for translating messages across providers """ -import base64 -import binascii -import io import json -import mimetypes -import re -from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence +from collections.abc import Hashable, Iterable, Mapping, Sequence from dataclasses import dataclass -from graphlib import CycleError, TopologicalSorter -from itertools import chain, groupby, islice -from os import PathLike -from pathlib import Path -from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast +from itertools import chain, groupby +from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast -from pydantic import TypeAdapter, ValidationError +from pydantic import TypeAdapter import litellm -from litellm import verbose_logger -from litellm.exceptions import UnsupportedParamsError -from litellm.litellm_core_utils.batch_utils import InMemoryFile -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data as extract_file_data, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_metadata as extract_file_metadata, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_file_ids_from_messages as get_file_ids_from_messages, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_format_from_file_id as get_format_from_file_id, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_image_mime_type_from_url as get_image_mime_type_from_url, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + infer_content_type_from_url_and_content as infer_content_type_from_url_and_content, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + update_messages_with_model_file_ids as update_messages_with_model_file_ids, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + update_responses_input_with_model_file_ids as update_responses_input_with_model_file_ids, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + update_responses_tools_with_model_file_ids as update_responses_tools_with_model_file_ids, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_lookaround_regex_patterns as drop_lookaround_regex_patterns, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns as drop_non_python_regex_patterns, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_combinators_and_drop_non_python_regex_patterns as flatten_combinators_and_drop_non_python_regex_patterns, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators as flatten_top_level_schema_combinators, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + is_object_schema as is_object_schema, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_fragment_ref as schema_fragment_ref, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_grammar_branch_count as schema_grammar_branch_count, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_has_lookaround_pattern as schema_has_lookaround_pattern, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_has_open_object as schema_has_open_object, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_has_unresolvable_ref as schema_has_unresolvable_ref, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_optional_property_count as schema_optional_property_count, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_property_count as schema_property_count, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_string_formats as schema_string_formats, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_union_property_count as schema_union_property_count, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + tool_with_sanitized_parameters as tool_with_sanitized_parameters, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + LITELLM_INTERNAL_MESSAGE_FIELDS as LITELLM_INTERNAL_MESSAGE_FIELDS, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS as LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT as PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + cache_control_as_prompt_cache_breakpoint as cache_control_as_prompt_cache_breakpoint, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + chat_completion_prompt_cache_breakpoint_for_wire as chat_completion_prompt_cache_breakpoint_for_wire, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + chat_completion_prompt_cache_breakpoints_for_wire as chat_completion_prompt_cache_breakpoints_for_wire, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + prompt_cache_breakpoint_as_cache_control as prompt_cache_breakpoint_as_cache_control, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + prompt_cache_breakpoint_for_wire as prompt_cache_breakpoint_for_wire, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + prompt_cache_breakpoint_with_readable_mode as prompt_cache_breakpoint_with_readable_mode, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + prompt_cache_breakpoint_within as prompt_cache_breakpoint_within, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + whole_prompt_cache_breakpoint_or_none as whole_prompt_cache_breakpoint_or_none, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + with_prompt_cache_breakpoint as with_prompt_cache_breakpoint, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + without_prompt_cache_breakpoints as without_prompt_cache_breakpoints, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + ENCRYPTED_REASONING_SIGNATURE_PREFIX as ENCRYPTED_REASONING_SIGNATURE_PREFIX, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + anthropic_content_lists as anthropic_content_lists, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + converse_redacted_content as converse_redacted_content, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + encrypted_content_from_signature as encrypted_content_from_signature, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + encrypted_content_of_block as encrypted_content_of_block, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + encrypted_reasoning_signature as encrypted_reasoning_signature, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + extract_reasoning_content as extract_reasoning_content, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + has_blank_redacted_data as has_blank_redacted_data, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + is_encrypted_reasoning_block as is_encrypted_reasoning_block, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + is_unsignable_thinking_block as is_unsignable_thinking_block, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + native_redacted_thinking_data as native_redacted_thinking_data, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + parse_content_for_reasoning as parse_content_for_reasoning, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + reasoning_content_from_thinking_blocks as reasoning_content_from_thinking_blocks, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + responses_reasoning_items_from_thinking_blocks as responses_reasoning_items_from_thinking_blocks, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + strip_encrypted_reasoning_from_messages as strip_encrypted_reasoning_from_messages, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( + MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS as MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( + parse_tool_call_arguments as parse_tool_call_arguments, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( + salvage_concatenated_tool_arguments as salvage_concatenated_tool_arguments, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( + split_concatenated_json_objects as split_concatenated_json_objects, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( + _image_part_url, # pyright: ignore[reportPrivateUsage] # Shared implementation for as_openai_image_part +) +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( + anthropic_image_source_to_openai_url as anthropic_image_source_to_openai_url, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( + embedding_input_is_all_media as embedding_input_is_all_media, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( + split_embedding_media_for_tpm as split_embedding_media_for_tpm, # noqa: PLC0414 # public compatibility re-export +) +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( + split_media_for_tpm as split_media_for_tpm, # noqa: PLC0414 # public compatibility re-export +) from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionAudioObject, - ChatCompletionCachedContent, ChatCompletionFileObject, ChatCompletionImageObject, - ChatCompletionPromptCacheBreakpoint, - ChatCompletionReasoningItem, - ChatCompletionReasoningSummaryTextBlock, - ChatCompletionRedactedThinkingBlock, ChatCompletionResponseMessage, ChatCompletionTextObject, - ChatCompletionThinkingBlock, ChatCompletionToolParam, ChatCompletionUserMessage, - PromptCacheBreakpoint, ) from litellm.types.utils import ( Choices, - ExtractedFileData, - FileTypes, ModelResponse, - SpecialEnums, StreamingChoices, ) @@ -58,6 +218,10 @@ DEFAULT_USER_CONTINUE_MESSAGE: Final = ChatCompletionUserMessage(content="Please DEFAULT_ASSISTANT_CONTINUE_MESSAGE: Final = ChatCompletionAssistantMessage(content="Please continue.", role="assistant") +_extract_reasoning_content: Final = extract_reasoning_content +_get_image_mime_type_from_url: Final = get_image_mime_type_from_url +_parse_content_for_reasoning: Final = parse_content_for_reasoning + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LoggingClass @@ -245,228 +409,6 @@ _IMAGE_CONTENT_PART_TYPES: Final = ("image_url", "input_image", "image", "comput _IMAGE_SCAN_MAX_DEPTH: Final = 4 -@dataclass(frozen=True, slots=True) -class _TPMMediaProjection: - content: object - media: tuple[Mapping[str, object], ...] = () - - -def _tpm_image_part(url: str, detail: object = "auto") -> Mapping[str, object]: - return {"type": "image_url", "image_url": {"url": url, "detail": detail}} - - -def _tpm_blank_fields(fields: Mapping[str, object], names: tuple[str, ...]) -> Mapping[str, object]: - return {key: "" if key in names and isinstance(value, str) else value for key, value in fields.items()} - - -def _tpm_file_media(part: Mapping[str, object]) -> _TPMMediaProjection: - from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type - - nested: Final = part.get("type") == "file" - fields: Final = as_str_mapping(part.get("file")) if nested else part - if fields is None: - return _TPMMediaProjection(part) - raw_data: Final = fields.get("file_data") - data: Final = raw_data if isinstance(raw_data, str) else "" - declared: Final = next( - ( - value - for key in ("format", "mime_type", "content_type") - if isinstance(value := fields.get(key), str) and value - ), - "", - ) - mime: Final = (declared or parse_mime_type(data[:256]) or "").split(";", 1)[0].strip().lower() - if not mime.startswith(("image/", "audio/")): - return _TPMMediaProjection(part) - raw_url: Final = fields.get("file_url") or fields.get("file_id") - url: Final = data or (raw_url if isinstance(raw_url, str) else "") - header_end: Final = data.find(",", 0, 256) - encoded: Final = ( - data[header_end + 1 :] - if header_end >= 0 and data[:header_end].endswith(";base64") - else "" - if data.startswith(("data:", "https://", "http://", "gs://")) - else data - ) - media: Final = ( - _tpm_image_part( - url - if not data or url.startswith(("data:", "https://", "http://", "gs://")) - else f"data:{mime};base64,{url}", - fields.get("detail", "auto"), - ) - if mime.startswith("image/") - else {"type": "input_audio", "input_audio": {"data": encoded, "format": mime.removeprefix("audio/")}} - ) - projected: Final = _tpm_blank_fields(fields, ("file_data", "file_url")) - return _TPMMediaProjection({**part, "file": projected} if nested else projected, (media,)) - - -def _tpm_media_leaf(value: object) -> _TPMMediaProjection: - part: Final = as_str_mapping(value) - if part is None: - return _TPMMediaProjection(value) - kind: Final = part.get("type") - if kind in ("file", "input_file"): - return _tpm_file_media(part) - if kind == "input_audio": - audio: Final = as_str_mapping(part.get("input_audio")) - return _TPMMediaProjection( - {**part, "input_audio": _tpm_blank_fields(audio, ("data",))} if audio is not None else value, - ({"type": "input_audio", "input_audio": part.get("input_audio")},), - ) - if kind in ("image_url", "input_image", "image", "computer_screenshot"): - image: Final = as_str_mapping(part.get("image_url")) - source: Final = as_str_mapping(part.get("source")) - canonical: Final = _tpm_image_part( - _image_part_url({**part, "type": "input_image"} if kind == "computer_screenshot" else part) or "", - (image or part).get("detail", part.get("detail", "auto")), - ) - projected: Final = ( - {**part, "image_url": _tpm_blank_fields(image, ("url",))} - if image is not None and kind in ("image_url", "input_image", "computer_screenshot") - else {**part, "source": _tpm_blank_fields(source, ("data", "url"))} - if source is not None and kind == "image" and source.get("type") in ("base64", "url") - else _tpm_blank_fields(part, ("image_url", "url")) - ) - return _TPMMediaProjection(projected, (canonical,)) - return _TPMMediaProjection(value) - - -def _tpm_media_list(value: object, split: Callable[[object], _TPMMediaProjection]) -> _TPMMediaProjection: - if not isinstance(value, (list, tuple)): - return _TPMMediaProjection(value) - parts: Final = cast(Sequence[object], value) # cast-ok: finite JSON content arrays - projected: Final = tuple(map(split, parts)) - media: Final = tuple(chain.from_iterable(part.media for part in projected)) - content: Final = tuple(part.content for part in projected) - return _TPMMediaProjection(parts if not media else content if isinstance(value, tuple) else list(content), media) - - -def _tpm_replace_field(part: Mapping[str, object], key: str, projection: _TPMMediaProjection) -> _TPMMediaProjection: - return _TPMMediaProjection({**part, key: projection.content} if projection.media else part, projection.media) - - -def _tpm_document_media(value: object) -> _TPMMediaProjection: - part: Final = as_str_mapping(value) - source: Final = as_str_mapping(part.get("source")) if part is not None else None - if part is None or part.get("type") != "document" or source is None or source.get("type") != "content": - return _tpm_media_leaf(value) - content: Final = _tpm_media_list(source.get("content"), _tpm_media_leaf) - return _tpm_replace_field(part, "source", _tpm_replace_field(source, "content", content)) - - -def _tpm_tool_result_media(value: object) -> _TPMMediaProjection: - part: Final = as_str_mapping(value) - if part is None or part.get("type") != "tool_result": - return _tpm_document_media(value) - return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_document_media)) - - -def _tpm_code_output_media(value: object) -> _TPMMediaProjection: - part: Final = as_str_mapping(value) - url: Final = part.get("url") if part is not None else None - if part is None or part.get("type") != "image" or not isinstance(url, str) or not url: - return _TPMMediaProjection(value) - return _TPMMediaProjection(_tpm_blank_fields(part, ("url",)), (_tpm_image_part(url),)) - - -def _tpm_input_media(value: object) -> _TPMMediaProjection: - part: Final = as_str_mapping(value) - if part is None: - return _TPMMediaProjection(value) - kind: Final = part.get("type") - if kind in ("function_call_output", "custom_tool_call_output"): - return _tpm_replace_field(part, "output", _tpm_media_list(part.get("output"), _tpm_tool_result_media)) - if kind == "computer_call_output": - return _tpm_replace_field(part, "output", _tpm_media_leaf(part.get("output"))) - if kind == "code_interpreter_call": - return _tpm_replace_field(part, "outputs", _tpm_media_list(part.get("outputs"), _tpm_code_output_media)) - result: Final = part.get("result") - if kind == "image_generation_call" and isinstance(result, str) and result: - return _TPMMediaProjection( - _tpm_blank_fields(part, ("result",)), - (_tpm_image_part("data:application/octet-stream;base64," + result),), - ) - if kind == "message" or (kind is None and isinstance(part.get("role"), str)): - return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_tool_result_media)) - return _tpm_tool_result_media(value) - - -def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]: - projection: Final = _tpm_media_list(value, _tpm_input_media) - return projection.content, projection.media - - -_TPM_EMBEDDING_FILE_MIME_PREFIXES: Final = ("application/pdf", "video/") - - -def _tpm_embedding_file(data_uri: str) -> tuple[Mapping[str, object], ...]: - from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type - - mime: Final = (parse_mime_type(data_uri[:256]) or "").strip().lower() - if not mime.startswith(_TPM_EMBEDDING_FILE_MIME_PREFIXES): - return () - return ({"type": "file", "file": {"file_data": data_uri, "format": mime}},) - - -_BASE64_PAYLOAD: Final = re.compile(r"[A-Za-z0-9+/]*={0,2}") - - -def _is_base64_data_uri(value: str) -> bool: - header_end: Final = value.find(",", 0, 256) - if header_end < 0 or not value[:header_end].endswith(";base64"): - return False - payload_start: Final = header_end + 1 - if (len(value) - payload_start) % 4 or _BASE64_PAYLOAD.fullmatch(value, payload_start) is None: - return False - last_quantum: Final = value[max(payload_start, len(value) - 4) :] - return base64.b64encode(base64.b64decode(last_quantum)).decode("ascii") == last_quantum - - -def _tpm_embedding_file_block(part: Mapping[str, object]) -> _TPMMediaProjection: - projection: Final = _tpm_file_media(part) - fields: Final = as_str_mapping(part.get("file")) - data: Final = fields.get("file_data") if fields is not None else None - if projection.media or fields is None or not isinstance(data, str) or not _is_base64_data_uri(data): - return projection - media: Final = _tpm_embedding_file(data) - return _TPMMediaProjection({**part, "file": _tpm_blank_fields(fields, ("file_data",))} if media else part, media) - - -def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection: - part: Final = as_str_mapping(value) - if part is not None and part.get("type") == "file": - return _tpm_embedding_file_block(part) - if not isinstance(value, str) or not value.startswith("data:"): - return _tpm_media_leaf(value) - if not _is_base64_data_uri(value): - return _TPMMediaProjection(value) - media: Final = _tpm_file_media({"file_data": value}).media or _tpm_embedding_file(value) - return _TPMMediaProjection("" if media else value, media) - - -def _is_json_array(value: object) -> bool: - return isinstance(value, (list, tuple)) - - -def _tpm_embedding_item(value: object) -> _TPMMediaProjection: - return _tpm_media_list(value, _tpm_embedding_leaf) if _is_json_array(value) else _tpm_embedding_leaf(value) - - -def split_embedding_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]: - projection: Final = ( - _tpm_media_list(value, _tpm_embedding_item) if _is_json_array(value) else _tpm_embedding_leaf(value) - ) - return projection.content, projection.media - - -def embedding_input_is_all_media(value: object) -> bool: - items: Final = cast(Sequence[object], value) if _is_json_array(value) else (value,) # cast-ok: JSON input array - return all(_tpm_embedding_leaf(item).media for item in items) - - def _content_parts_contain_image(parts: Sequence[object]) -> bool: """Depth-bounded frontier walk over nested content lists, iterative because the repo bans recursion; an Anthropic tool_result nests its image parts exactly one level down.""" @@ -487,19 +429,6 @@ def _content_parts_contain_image(parts: Sequence[object]) -> bool: return False -def anthropic_image_source_to_openai_url(image_source: Mapping[str, object]) -> str | None: - """Data or remote URL for an Anthropic ``source`` block, in the form chat completions expects.""" - source_type: Final = image_source.get("type") - if source_type == "base64": - media_type: Final = image_source.get("media_type") or "image/jpeg" - image_data: Final = image_source.get("data") or "" - return f"data:{media_type};base64,{image_data}" if image_data else None - if source_type == "url": - url: Final = image_source.get("url") - return url if isinstance(url, str) else "" - return None - - @dataclass(frozen=True, slots=True) class InlineDocument: filename: str @@ -603,22 +532,6 @@ def messages_with_file_parts_as_image_urls(messages: Sequence[AllMessageValues]) return tuple(_typed_message_with_file_parts_as_image_urls(message) for message in messages) -def _image_part_url(part: Mapping[str, object]) -> str | None: - """The image URL carried by one content part, whichever of the three dialects wrote it.""" - part_type: Final = part.get("type") - if part_type in ("image_url", "input_image"): - image_url: Final = part.get("image_url") or part.get("url") - if isinstance(image_url, str): - return image_url - image_fields: Final = as_str_mapping(image_url) - url: Final = image_fields.get("url") if image_fields is not None else None - return url if isinstance(url, str) else None - if part_type == "image": - source: Final = part.get("source") - return anthropic_image_source_to_openai_url(source) if isinstance(source, Mapping) else None - return None - - def as_openai_image_part(part: Mapping[str, object]) -> ChatCompletionImageObject | None: """One image content part rewritten into chat-completions dialect, or None when it is not one. @@ -867,433 +780,6 @@ def get_completion_messages( return messages -def get_format_from_file_id(file_id: str | None) -> str | None: - """ - Gets format from file id - - unified_file_id = litellm_proxy:{};unified_id,{} - If not a unified file id, returns 'file' as default format - """ - from litellm.proxy.openai_files_endpoints.common_utils import ( - convert_b64_uid_to_unified_uid, - ) - - if not file_id: - return None - try: - transformed_file_id: Final = convert_b64_uid_to_unified_uid(file_id) - if transformed_file_id.startswith(SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value): - match: Final = re.match( - f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}:(.*?);unified_id", - transformed_file_id, - ) - if match: - return match.group(1) - - return None - except Exception: - return None - - -def update_messages_with_model_file_ids( - messages: list[AllMessageValues], - model_id: str | None, - model_file_id_mapping: dict[str, dict[str, str]], -) -> list[AllMessageValues]: - """ - Updates messages with model file ids. - - For managed files (unified file IDs), uses model_file_id_mapping if it - resolves the id, otherwise decodes the base64-encoded unified file ID - and extracts the llm_output_file_id directly. Mirrors the Responses-API - sibling `update_responses_input_with_model_file_ids`. - - model_file_id_mapping: Dict[str, Dict[str, str]] = { - "litellm_proxy/file_id": { - "model_id": "provider_file_id" - } - } - """ - from litellm.proxy.openai_files_endpoints.common_utils import ( - convert_b64_uid_to_unified_uid, - get_original_file_id, - is_base64_encoded_unified_file_id, - is_model_embedded_id, - ) - - for message in messages: - if message.get("role") == "user": - content = message.get("content") - if content: - if isinstance(content, str): - continue - for c in content: - if not isinstance(c, dict): - # Content list items aren't always dicts. e.g. - # text_completion forwards a token-ids list/list-of- - # lists through this path. Skip non-dict items - # instead of indexing into them. - continue - if c.get("type") == "file": - file_object = cast(ChatCompletionFileObject, c) - file_object_file_field = file_object.get("file") - if not isinstance(file_object_file_field, dict): - # Content block has `type: "file"` but not the - # OpenAI Chat Completions shape (e.g. a LangChain - # v1 standardized file block, or a provider-native - # shape that also uses `type: "file"`). Nothing to - # remap here, so skip instead of crashing. - continue - file_id = file_object_file_field.get("file_id") - format = file_object_file_field.get("format", get_format_from_file_id(file_id)) - - if file_id: - provider_file_id = ( - model_file_id_mapping.get(file_id, {}).get(model_id) - if model_file_id_mapping and model_id is not None - else None - ) - if not provider_file_id and is_base64_encoded_unified_file_id(file_id): - unified_file_id = convert_b64_uid_to_unified_uid(file_id) - if "llm_output_file_id," in unified_file_id: - provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0] - if not provider_file_id and is_model_embedded_id(file_id): - provider_file_id = get_original_file_id(file_id) - file_object_file_field["file_id"] = provider_file_id or file_id - if format: - file_object_file_field["format"] = format - return messages - - -def update_responses_input_with_model_file_ids( - input: object, - model_id: str | None = None, - model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None, -) -> object: - """ - Updates responses API input with provider-specific file IDs. - File IDs are always inside the content array, not as direct input_file items. - - For managed files (unified file IDs), uses model_file_id_mapping if provided, - otherwise decodes the base64-encoded unified file ID and extracts the llm_output_file_id directly. - - Args: - input: The responses API input parameter - model_id: The model ID to use for looking up provider-specific file IDs - model_file_id_mapping: Dictionary mapping litellm file IDs to provider file IDs - Format: {"litellm_file_id": {"model_id": "provider_file_id"}} - """ - from litellm.proxy.openai_files_endpoints.common_utils import ( - convert_b64_uid_to_unified_uid, - get_original_file_id, - is_base64_encoded_unified_file_id, - is_model_embedded_id, - ) - - if isinstance(input, str): - return input - - if not isinstance(input, list): - return input - - updated_input: Final = [] - for item in input: - if not isinstance(item, dict): - updated_input.append(item) - continue - - updated_item = item.copy() - content = item.get("content") - if isinstance(content, list): - updated_content = [] - for content_item in content: - if isinstance(content_item, dict) and content_item.get("type") == "input_file": - file_id = content_item.get("file_id") - if isinstance(file_id, str) and file_id: - provider_file_id = file_id # Default to original - - # Check if we have a mapping for this file ID - if model_file_id_mapping and model_id and file_id in model_file_id_mapping: - # Use the model-specific file ID from mapping - provider_file_id = model_file_id_mapping.get(file_id, {}).get(model_id) or file_id - updated_content_item = content_item.copy() - updated_content_item["file_id"] = provider_file_id - updated_content.append(updated_content_item) - else: - # Check if this is a base64-encoded unified file ID without mapping - is_unified_file_id = is_base64_encoded_unified_file_id(file_id) - if is_unified_file_id: - # Fallback: decode unified file ID - unified_file_id = convert_b64_uid_to_unified_uid(file_id) - if "llm_output_file_id," in unified_file_id: - provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0] - - updated_content_item = content_item.copy() - updated_content_item["file_id"] = provider_file_id - updated_content.append(updated_content_item) - elif is_model_embedded_id(file_id): - updated_content_item = content_item.copy() - updated_content_item["file_id"] = get_original_file_id(file_id) - updated_content.append(updated_content_item) - else: - # Not a managed file, keep as-is - updated_content.append(content_item) - else: - updated_content.append(content_item) - else: - updated_content.append(content_item) - updated_item["content"] = updated_content - - updated_input.append(updated_item) - - return updated_input - - -def _decode_vector_store_ids_in_tools( - tools: list[dict[str, object]] | None, -) -> list[dict[str, object]] | None: - """ - Decodes unified (LiteLLM-managed) vector_store_ids in file_search tools to - provider-native IDs. Non-unified IDs are passed through unchanged. - - This runs unconditionally — no file-ID mapping is required. - """ - if not tools or not isinstance(tools, list): - return tools - - from litellm.llms.base_llm.managed_resources.utils import ( - is_base64_encoded_unified_id, - parse_unified_id, - ) - - updated_tools: Final = [] - for tool in tools: - if not isinstance(tool, dict) or tool.get("type") != "file_search": - updated_tools.append(tool) - continue - - vector_store_ids = tool.get("vector_store_ids") - if not isinstance(vector_store_ids, list): - updated_tools.append(tool) - continue - - decoded_ids = [] - for vs_id in vector_store_ids: - if not isinstance(vs_id, str) or not is_base64_encoded_unified_id(vs_id): - decoded_ids.append(vs_id) - continue - - parsed = parse_unified_id(vs_id) - provider_resource_id = parsed.get("provider_resource_id") if parsed else None - - if not provider_resource_id: - verbose_logger.warning( - "file_search tool contains unified vector_store_id '%s' that could " - "not be decoded to a provider resource ID — passing original ID. " - "Ensure the vector store was created via LiteLLM.", - vs_id, - ) - decoded_ids.append(vs_id) - else: - decoded_ids.append(provider_resource_id) - - updated_tools.append({**tool, "vector_store_ids": decoded_ids}) - - return updated_tools - - -def update_responses_tools_with_model_file_ids( - tools: list[dict[str, object]] | None, - model_id: str | None = None, - model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None, -) -> list[dict[str, object]] | None: - """ - Updates responses API tools with provider-specific file IDs. - - Pass 1 (always): decode unified vector_store_ids in file_search tools. - Pass 2 (needs mapping): map code_interpreter container file_ids to provider IDs. - - Args: - tools: The responses API tools parameter - model_id: The model ID to use for looking up provider-specific file IDs - model_file_id_mapping: Dictionary mapping litellm file IDs to provider file IDs - Format: {"litellm_file_id": {"model_id": "provider_file_id"}} - """ - if not tools or not isinstance(tools, list): - return tools - - # Pass 1: decode unified vector_store_ids (no mapping needed) - tools = _decode_vector_store_ids_in_tools(tools) or tools - - # Pass 2: map code_interpreter file IDs (requires mapping) - if not model_file_id_mapping or not model_id: - return tools - - updated_tools: Final = [] - for tool in tools: - if not isinstance(tool, dict): - updated_tools.append(tool) - continue - - updated_tool = tool.copy() - - # Handle code_interpreter with container file_ids - if tool.get("type") == "code_interpreter": - container = tool.get("container") - if isinstance(container, dict): - container_file_ids = container.get("file_ids") - if isinstance(container_file_ids, list): - updated_file_ids = [] - for file_id in container_file_ids: - if isinstance(file_id, str): - # Check if we have a mapping for this file ID - if file_id in model_file_id_mapping: - # Map to provider-specific file ID - provider_file_id = model_file_id_mapping.get(file_id, {}).get(model_id) or file_id - updated_file_ids.append(provider_file_id) - else: - updated_file_ids.append(file_id) - else: - updated_file_ids.append(file_id) - - # Update the tool with new file IDs - updated_container = container.copy() - updated_container["file_ids"] = updated_file_ids - updated_tool["container"] = updated_container - - updated_tools.append(updated_tool) - - return updated_tools - - -def extract_file_metadata(file_data: FileTypes) -> tuple[str | None, str | None]: - """ - Resolve (filename, content_type) without reading the file body. - - Mirrors extract_file_data's metadata resolution but never calls .read(), so - it stays O(1) on large uploads. Use this when only metadata is needed (batch - detection, GCS object naming) and the body must remain a streamable Path/handle. - """ - filename: str | None = None - content_type: str | None = None - file_content: object = None - - if isinstance(file_data, tuple): - if len(file_data) == 2: - filename, file_content = file_data - elif len(file_data) == 3: - filename, file_content, content_type = file_data - elif len(file_data) == 4: - filename, file_content, content_type, _ = file_data - elif isinstance(file_data, InMemoryFile): - filename = file_data.name - content_type = file_data.content_type - else: - file_content = file_data - - if filename is None: - if isinstance(file_content, PathLike): - filename = Path(file_content).name - elif isinstance(file_content, io.IOBase): - name_attr: Final = getattr(file_content, "name", None) - if isinstance(name_attr, str): - filename = Path(name_attr).name - - if not content_type: - guessed: Final = mimetypes.guess_type(filename)[0] if filename else None - content_type = guessed or "application/octet-stream" - - return filename, content_type - - -def extract_file_data(file_data: FileTypes) -> ExtractedFileData: - """ - Extracts and processes file data from various input formats. - - Args: - file_data: Can be a tuple of (filename, content, [content_type], [headers]) or direct file content - - Returns: - ExtractedFileData containing: - - filename: Name of the file if provided - - content: The file content in bytes - - content_type: MIME type of the file - - headers: Any additional headers - """ - # Parse the file_data based on its type - filename = None - file_content = None - content_type = None - file_headers: Mapping[str, str] = {} - - if isinstance(file_data, tuple): - if len(file_data) == 2: - filename, file_content = file_data - elif len(file_data) == 3: - filename, file_content, content_type = file_data - elif len(file_data) == 4: - filename, file_content, content_type, file_headers = file_data - elif isinstance(file_data, InMemoryFile): - filename = file_data.name - file_content = file_data - content_type = file_data.content_type - else: - file_content = file_data - # Convert content to bytes - if isinstance(file_content, str): - # Bare string inputs are rejected: when this helper runs in a proxy - # request handler the string came from an attacker-controlled form - # field, and opening it as a path is an arbitrary file read on the - # proxy host. SDK callers who want to upload from a path should - # either pass a pathlib.Path (a PathLike instance — see the branch - # below) or open the file themselves and pass the handle / bytes. - raise ValueError( - "extract_file_data does not accept bare str inputs. Pass bytes, " - "an open file handle, a (filename, content) tuple, or a " - "pathlib.Path. To upload a local file from a path, call " - "open(path, 'rb') yourself." - ) - if isinstance(file_content, PathLike): - # PathLike (pathlib.Path) is a Python-level type that HTTP form - # values can't fabricate. Treat as a local file path for SDK - # convenience. - if filename is None: - filename = Path(file_content).name - with open(file_content, "rb") as f: - content = f.read() - elif isinstance(file_content, io.IOBase): - # If it's a file-like object - # Try to get filename from file handle if not already set - if not filename and hasattr(file_content, "name"): - filename = Path(file_content.name).name - - content = file_content.read() - - if isinstance(content, str): - content = content.encode("utf-8") - # Reset file pointer to beginning - file_content.seek(0) - elif isinstance(file_content, bytes): - content = file_content - else: - raise ValueError(f"Unsupported file content type: {type(file_content)}") - - # Use provided content type or guess based on filename - if not content_type: - if filename: - guessed_type: Final = mimetypes.guess_type(filename)[0] - content_type = guessed_type if guessed_type else "application/octet-stream" - else: - content_type = "application/octet-stream" - - return ExtractedFileData( - filename=filename, - content=content, - content_type=content_type, - headers=file_headers, - ) - - # --------------------------------------------------------------------------- # Generic, dependency-free implementation of `unpack_defs` # --------------------------------------------------------------------------- @@ -1536,633 +1022,6 @@ def sanitize_input_schema_for_anthropic(input_schema: dict) -> "AnthropicInputSc return AnthropicInputSchema(**filtered) -_TOP_LEVEL_SCHEMA_COMBINATORS: Final = ("allOf", "anyOf", "oneOf") -_OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS: Final = ("enum", "const", "not") -_LOCAL_SCHEMA_REF_PREFIXES: Final = (("#/$defs/", "$defs"), ("#/definitions/", "definitions")) -_MAX_SCHEMA_FLATTEN_DEPTH: Final = 32 -_EMPTY_SCHEMA: Final[Mapping[str, object]] = MappingProxyType({}) - - -def _schema_properties(schema: Mapping[str, object]) -> Mapping[str, object]: - properties: Final = schema.get("properties") - return properties if isinstance(properties, dict) else _EMPTY_SCHEMA - - -def _schema_branches(schema: Mapping[str, object], combinator: str) -> tuple[object, ...]: - branches: Final = schema.get(combinator) - return tuple(branches) if isinstance(branches, list) else () - - -def _schema_required_names(schema: Mapping[str, object]) -> frozenset[str]: - required: Final = schema.get("required") - if not isinstance(required, list): - return frozenset() - return frozenset(name for name in required if isinstance(name, str)) - - -def _combinator_required_names(combinator: str, branches: tuple[Mapping[str, object], ...]) -> frozenset[str]: - branch_names: Final = tuple(_schema_required_names(branch) for branch in branches) - if not branch_names: - return frozenset() - if combinator == "allOf": - return branch_names[0].union(*branch_names[1:]) - return branch_names[0].intersection(*branch_names[1:]) - - -def _resolve_local_schema_ref(root: Mapping[str, object], ref: str) -> Mapping[str, object] | None: - matched: Final = next( - ((prefix, container) for prefix, container in _LOCAL_SCHEMA_REF_PREFIXES if ref.startswith(prefix)), - None, - ) - if matched is None: - return None - prefix, container = matched - definitions: Final = root.get(container) - if not isinstance(definitions, dict): - return None - target: Final = definitions.get(ref[len(prefix) :]) - return target if isinstance(target, dict) else None - - -def _mergeable_branch( - root: Mapping[str, object], - branch: object, - seen_refs: frozenset[str], - depth: int, - expanded_refs: dict[str, Mapping[str, object] | None], # mutable-ok: per-call memo bounding repeated $ref work -) -> Mapping[str, object] | None: - if not isinstance(branch, dict) or depth > _MAX_SCHEMA_FLATTEN_DEPTH: - return None - ref: Final = branch.get("$ref") - if not isinstance(ref, str): - flattened: Final = _flatten_schema_against_root(branch, root, seen_refs, depth, expanded_refs) - if any(combinator in flattened for combinator in _TOP_LEVEL_SCHEMA_COMBINATORS): - return None - return flattened - if ref in expanded_refs: - return expanded_refs[ref] - if ref in seen_refs: - return None - target: Final = _resolve_local_schema_ref(root, ref) - expanded: Final = ( - None - if target is None - else _mergeable_branch(root, target, seen_refs | frozenset((ref,)), depth + 1, expanded_refs) - ) - expanded_refs[ref] = expanded - return expanded - - -def _is_object_schema(schema: Mapping[str, object]) -> bool: - return schema.get("type") == "object" or ("type" not in schema and "properties" in schema) - - -def _flatten_schema_against_root( - schema: Mapping[str, object], - root: Mapping[str, object], - seen_refs: frozenset[str], - depth: int, - expanded_refs: dict[str, Mapping[str, object] | None], # mutable-ok: per-call memo bounding repeated $ref work -) -> Mapping[str, object]: - raw_branch_groups: Final = tuple( - ( - combinator, - tuple( - _mergeable_branch(root, branch, seen_refs, depth + 1, expanded_refs) - for branch in _schema_branches(schema, combinator) - ), - ) - for combinator in _TOP_LEVEL_SCHEMA_COMBINATORS - if isinstance(schema.get(combinator), list) - ) - dropped: Final = ( - *(combinator for combinator, _ in raw_branch_groups), - *(key for key in _OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS if key in schema), - ) - if not dropped: - return schema - - if any(branch is None for _, group in raw_branch_groups for branch in group): - return schema - branch_groups: Final = tuple( - (combinator, tuple(branch for branch in group if branch is not None)) for combinator, group in raw_branch_groups - ) - branches: Final = tuple(branch for _, group in branch_groups for branch in group) - is_object_schema: Final = _is_object_schema(schema) or ( - "type" not in schema and branches != () and all(_is_object_schema(branch) for branch in branches) - ) - if not is_object_schema: - return schema - - merged_properties: Final = { - name: value for source in (*reversed(branches), schema) for name, value in _schema_properties(source).items() - } - required_names: Final = _schema_required_names(schema).union( - *(_combinator_required_names(combinator, group) for combinator, group in branch_groups) - ) - kept: Final = MappingProxyType({key: value for key, value in schema.items() if key not in dropped}) - required_update: Final = MappingProxyType({"required": sorted(required_names)}) if required_names else _EMPTY_SCHEMA - return { - **kept, - "type": "object", - "properties": merged_properties, - **required_update, - } - - -def flatten_top_level_schema_combinators(schema: Mapping[str, object]) -> Mapping[str, object]: - """Merge top-level ``allOf``/``anyOf``/``oneOf`` branches into an object tool schema. - - OpenAI's function-calling validator rejects tool ``parameters`` carrying - 'oneOf'/'anyOf'/'allOf'/'enum'/'const'/'not' at the top level (nested uses - are accepted), while lenient backends such as the ChatGPT backend Codex - talks to natively accept them, so an MCP tool declaring a top-level union - 400s through LiteLLM. Branch properties merge without clobbering (the - top-level schema wins, then earlier branches); ``required`` becomes the - top-level list plus the intersection of the branch lists for anyOf/oneOf - or their union for allOf. Branches that are local ``$ref``s - (``#/$defs/...`` or ``#/definitions/...``) are resolved first, each ref - at most once per call, and branches that are themselves combinators are - flattened recursively up to a fixed depth; a branch that cannot be fully - merged (a boolean schema, an external or cyclic ``$ref``, a non-object - union, or nesting past the depth cap) leaves the whole schema untouched so - OpenAI's own validation still applies. Non-object schemas pass through - unchanged and the input is never mutated. - """ - return _flatten_schema_against_root(schema, schema, frozenset(), 0, {}) - - -_SUBSCHEMA_KEYWORDS: Final = frozenset( - { - "additionalItems", - "additionalProperties", - "contains", - "else", - "if", - "items", - "not", - "propertyNames", - "then", - "unevaluatedItems", - "unevaluatedProperties", - } -) -_SUBSCHEMA_LIST_KEYWORDS: Final = frozenset({"allOf", "anyOf", "items", "oneOf", "prefixItems"}) -_SUBSCHEMA_MAP_KEYWORDS: Final = frozenset( - {"$defs", "definitions", "dependentSchemas", "patternProperties", "properties"} -) - -_MAX_SCHEMA_NESTING: Final = 1024 - - -def drop_non_python_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]: - """Drop every regex in a schema position that Python's ``re`` cannot compile. - - OpenAI validates tool ``parameters`` against the 2020-12 metaschema with - ``jsonschema``'s format checker, which hands each ``pattern`` value and each - ``patternProperties`` key to ``re.compile``, so a regex written for an - ECMA-262 engine (Unicode property escapes such as ``\\p{Cc}``, as in Claude - Code's ``Artifact`` tool) is refused with "'...' is not a 'regex'" by every - model family on both the chat and Responses wires. Only schema positions are - walked (properties, items, combinators, ``$defs`` and the other applicators), - so a ``pattern`` key inside ``default``, ``examples``, ``const`` or vendor - extensions is data and stays. Outside strict mode the keyword is only a - hint, so dropping it costs the model a constraint and the caller nothing. - Compilable regexes and everything else pass through, the input is never - mutated, and the same object comes back when nothing was dropped. The walk - is level-order rather than recursive, rebuilt deepest level first, and stops - at more schema levels than a JSON parser admits, so a cyclic schema built in - code cannot spin it. - """ - return _schema_without_rejected_regex(schema, _is_not_python_regex) - - -def drop_lookaround_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]: - """Drop every regex in a schema position that uses a lookaround assertion. - - Some Bedrock Converse families compile tool schema regexes with an engine that - has no lookahead or lookbehind and refuse the whole request over one. The ``(?=``, - ``(?!``, ``(?<=`` and ``(? Mapping[str, object]: - rebuilt: dict[int, Mapping[str, object]] = {} # mutable-ok: per-call memo of rewritten nodes, deepest level first - for level in reversed(tuple(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING))): - rebuilt.update( - (id(node), rewritten) - for node in level - if (rewritten := _node_without_rejected_regex(node, rebuilt, rejected)) is not node - ) - return rebuilt.get(id(schema), schema) - - -def _schema_levels(schema: Mapping[str, object]) -> Iterator[tuple[Mapping[str, object], ...]]: - frontier: tuple[Mapping[str, object], ...] = (schema,) # rebind-ok: level-order cursor, one level a round - while frontier: - yield frontier - frontier = tuple(child for node in frontier for child in _subschemas(node)) - - -def schema_fragment_ref(ref: str) -> str: - return f"#{ref}" if ref.startswith(("/$defs/", "/definitions/")) else ref - - -def schema_has_unresolvable_ref(schema: Mapping[str, object]) -> bool: - definitions: Final = MappingProxyType(dict(_root_definitions(schema))) - if not _refs_in(schema) <= definitions.keys(): - return True - try: - tuple(TopologicalSorter({pointer: _refs_in(body) for pointer, body in definitions.items()}).static_order()) - except CycleError: - return True - return False - - -def schema_union_property_count(schema: Mapping[str, object]) -> int: - return sum(1 for _, node in _property_schemas(schema) if _is_union(node)) - - -def schema_optional_property_count(schema: Mapping[str, object]) -> int: - return sum(len(_optional_property_names(node)) for node in _schema_nodes(schema)) - - -def schema_property_count(schema: Mapping[str, object]) -> int: - return sum(len(properties) for node in _schema_nodes(schema) if (properties := _schema_at(node, "properties"))) - - -def schema_grammar_branch_count(schema: Mapping[str, object]) -> int: - return sum(_grammar_branches(node) for node in _schema_nodes(schema)) - - -def schema_has_open_object(schema: Mapping[str, object]) -> bool: - return any(_is_open_object(node) for node in _schema_nodes(schema)) - - -def schema_has_lookaround_pattern(schema: Mapping[str, object]) -> bool: - return any( - isinstance(pattern := node.get("pattern"), str) and _uses_regex_lookaround(pattern) - for node in _schema_nodes(schema) - ) - - -def schema_string_formats(schema: Mapping[str, object]) -> frozenset[str]: - return frozenset( - value for node in _schema_nodes(schema) if _may_be_string(node) and isinstance(value := node.get("format"), str) - ) - - -def _may_be_string(node: Mapping[str, object]) -> bool: - node_type: Final = node.get("type") - return node_type is None or node_type == "string" or "string" in _list_at(node, "type") - - -def is_object_schema(node: Mapping[str, object]) -> bool: - node_type: Final = node.get("type") - return node_type == "object" or (isinstance(node_type, list) and "object" in node_type) - - -def _schema_nodes(schema: Mapping[str, object]) -> Iterator[Mapping[str, object]]: - return chain.from_iterable(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING)) - - -_SCHEMA_LIST: Final = TypeAdapter(tuple[object, ...]) - - -def _list_at(container: Mapping[str, object], key: str) -> tuple[object, ...]: - value: Final = container.get(key) - return _SCHEMA_LIST.validate_python(value) if isinstance(value, list) else () - - -def _named_schemas(container: Mapping[str, object] | None) -> Iterator[tuple[str, Mapping[str, object]]]: - if container is None: - return iter(()) - return ((key, value) for key in container if (value := _schema_at(container, key)) is not None) - - -def _root_definitions(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]: - for defs_key in ("$defs", "definitions"): - yield from ((f"#/{defs_key}/{name}", body) for name, body in _named_schemas(_schema_at(schema, defs_key))) - - -def _refs_in(schema: Mapping[str, object]) -> frozenset[str]: - return frozenset( - schema_fragment_ref(ref) for node in _schema_nodes(schema) if isinstance(ref := node.get("$ref"), str) - ) - - -def _property_schemas(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]: - return chain.from_iterable(_named_schemas(_schema_at(node, "properties")) for node in _schema_nodes(schema)) - - -def _is_union(node: Mapping[str, object]) -> bool: - return isinstance(node.get("anyOf"), list) or isinstance(node.get("oneOf"), list) or len(_list_at(node, "type")) > 1 - - -def _grammar_branches(node: Mapping[str, object]) -> int: - properties: Final = _schema_at(node, "properties") - if properties is None: - return 0 - optional: Final = len(_optional_property_names(node)) - return (1 << optional) * (len(properties) - optional + 1) - - -def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]: - properties: Final = _schema_at(node, "properties") - if properties is None: - return frozenset() - return frozenset(properties) - frozenset(name for name in _list_at(node, "required") if isinstance(name, str)) - - -def _is_open_object(node: Mapping[str, object]) -> bool: - additional_properties: Final = node.get("additionalProperties") - if additional_properties is True or isinstance(additional_properties, dict) or "patternProperties" in node: - return True - return is_object_schema(node) and "properties" not in node and additional_properties is not False - - -def _subschemas(node: Mapping[str, object]) -> Iterator[Mapping[str, object]]: - for key, value in node.items(): - if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict): - yield from (sub for sub in value.values() if isinstance(sub, dict)) - elif key in _SUBSCHEMA_LIST_KEYWORDS and isinstance(value, list): - yield from (sub for sub in value if isinstance(sub, dict)) - elif key in _SUBSCHEMA_KEYWORDS and isinstance(value, dict): - yield value - - -def _node_without_rejected_regex( - node: Mapping[str, object], - rebuilt: Mapping[int, Mapping[str, object]], - rejected: Callable[[str], bool], -) -> Mapping[str, object]: - kept: Final = { - key: _keyword_value_rebuilt(key, value, rebuilt, rejected) - for key, value in node.items() - if key != "pattern" or not isinstance(value, str) or not rejected(value) - } - if len(kept) == len(node) and all(kept[key] is node[key] for key in kept): - return node - dropped_pattern_properties: Final = _dropped_pattern_properties(node, kept, rebuilt) - if not dropped_pattern_properties or kept.get("additionalProperties") is not False: - return kept - return {**kept, "additionalProperties": _any_of(dropped_pattern_properties)} - - -def _dropped_pattern_properties( - node: Mapping[str, object], - kept: Mapping[str, object], - rebuilt: Mapping[int, Mapping[str, object]], -) -> tuple[object, ...]: - before: Final = _schema_at(node, "patternProperties") - after: Final = _schema_at(kept, "patternProperties") - if before is None or after is None: - return () - return tuple(rebuilt.get(id(sub), sub) for name, sub in before.items() if name not in after) - - -def _schema_at(container: Mapping[str, object], key: str) -> Mapping[str, object] | None: - value: Final = container.get(key) - return value if isinstance(value, dict) else None - - -def _any_of(schemas: tuple[object, ...]) -> object: - return schemas[0] if len(schemas) == 1 else {"anyOf": list(schemas)} - - -def _keyword_value_rebuilt( - key: str, - value: object, - rebuilt: Mapping[int, Mapping[str, object]], - rejected: Callable[[str], bool], -) -> object: - if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict): - kept: Final = { - name: rebuilt.get(id(sub), sub) - for name, sub in value.items() - if key != "patternProperties" or not isinstance(name, str) or not rejected(name) - } - return value if len(kept) == len(value) and all(kept[name] is value[name] for name in kept) else kept - if key in _SUBSCHEMA_LIST_KEYWORDS and isinstance(value, list): - items: Final = [rebuilt.get(id(sub), sub) for sub in value] - return value if all(new is old for new, old in zip(items, value, strict=True)) else items - if key in _SUBSCHEMA_KEYWORDS and isinstance(value, dict): - return rebuilt.get(id(value), value) - return value - - -def _is_not_python_regex(pattern: str) -> bool: - try: - re.compile(pattern) - except (re.error, RecursionError): - return True - return False - - -_REGEX_LOOKAROUND_RE: Final = re.compile(r"\(\? bool: - return _REGEX_LOOKAROUND_RE.search(pattern) is not None - - -def flatten_combinators_and_drop_non_python_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]: - return flatten_top_level_schema_combinators(drop_non_python_regex_patterns(schema)) - - -def tool_with_sanitized_parameters( - tool: Mapping[str, object], - sanitize: Callable[[Mapping[str, object]], Mapping[str, object]], -) -> Mapping[str, object]: - """Run the tool's JSON schema through ``sanitize``: ``function.parameters`` on an - OpenAI tool, ``input_schema`` on an Anthropic one. The same object comes back when - nothing changed.""" - function: Final = _schema_at(tool, "function") - if function is not None: - parameters: Final = _schema_at(function, "parameters") - if parameters is None: - return tool - sanitized_parameters: Final = sanitize(parameters) - if sanitized_parameters is parameters: - return tool - return {**tool, "function": {**function, "parameters": sanitized_parameters}} - input_schema: Final = _schema_at(tool, "input_schema") - if input_schema is None: - return tool - sanitized_schema: Final = sanitize(input_schema) - return tool if sanitized_schema is input_schema else {**tool, "input_schema": sanitized_schema} - - -def get_image_mime_type_from_url(url: str) -> str | None: - """ - Get mime type for common image URLs - See gemini mime types: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/image-understanding#image-requirements - - Supported by Gemini: - application/pdf - audio/mpeg - audio/mp3 - audio/wav - audio/ogg - image/png - image/jpeg - image/webp - text/plain - video/mov - video/mpeg - video/mp4 - video/mpg - video/avi - video/wmv - video/mpegps - video/flv - """ - from urllib.parse import urlparse - - url = url.lower() - - # Parse URL to extract path without query parameters - # This handles URLs like: https://example.com/image.jpg?signature=... - parsed: Final = urlparse(url) - path: Final = parsed.path - - # Map file extensions to mime types - mime_types: Final = { - # Images - (".jpg", ".jpeg"): "image/jpeg", - (".png",): "image/png", - (".webp",): "image/webp", - # Videos - (".mp4",): "video/mp4", - (".mov",): "video/mov", - (".mpeg", ".mpg"): "video/mpeg", - (".avi",): "video/avi", - (".wmv",): "video/wmv", - (".mpegps",): "video/mpegps", - (".flv",): "video/flv", - # Audio - (".mp3",): "audio/mp3", - (".wav",): "audio/wav", - (".mpeg",): "audio/mpeg", - (".ogg",): "audio/ogg", - # Documents - (".pdf",): "application/pdf", - (".txt",): "text/plain", - } - - # Check each extension group against the URL - for extensions, mime_type in mime_types.items(): - if any(path.endswith(ext) for ext in extensions): - return mime_type - - return None - - -_get_image_mime_type_from_url = get_image_mime_type_from_url - - -def infer_content_type_from_url_and_content( - url: str, - content: bytes, - current_content_type: str | None = None, -) -> str: - """ - Infer content type from URL extension and binary content when content-type header is missing or generic. - - This helper implements a fallback strategy for determining MIME types when HTTP headers - are missing or provide generic values (like binary/octet-stream). It's commonly used - when processing images and documents from various sources (S3, URLs, etc.). - - Fallback Strategy: - 1. If current_content_type is valid (not None and not generic octet-stream), return it - 2. Try to infer from URL extension (handles query parameters) - 3. Try to detect from binary content signature (magic bytes) - 4. Raise ValueError if all methods fail - - Args: - url: The URL of the content (used to extract file extension) - content: The binary content (first ~100 bytes are sufficient for detection) - current_content_type: The current content-type from headers (may be None or generic) - - Returns: - str: The inferred MIME type (e.g., "image/png", "application/pdf") - - Raises: - ValueError: If content type cannot be determined by any method - - Example: - >>> content_type = infer_content_type_from_url_and_content( - ... url="https://s3.amazonaws.com/bucket/image.png?AWSAccessKeyId=123", - ... content=png_binary_data, - ... current_content_type="binary/octet-stream" - ... ) - >>> print(content_type) - "image/png" - """ - from litellm.litellm_core_utils.token_counter import get_image_type - - media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None - - # If we have a valid content type that's not generic, use it - if media_type and media_type not in [ - "binary/octet-stream", - "application/octet-stream", - ]: - return media_type - - # Extension to MIME type mapping - # Supports images, documents, and other common file types - extension_to_mime: Final = { - # Image formats - "jpg": "image/jpeg", - "jpeg": "image/jpeg", - "png": "image/png", - "gif": "image/gif", - "webp": "image/webp", - # Document formats - "pdf": "application/pdf", - "csv": "text/csv", - "doc": "application/msword", - "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "xls": "application/vnd.ms-excel", - "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "html": "text/html", - "txt": "text/plain", - "md": "text/markdown", - } - - # Try to infer from URL extension - if url: - extension: Final = url.split(".")[-1].lower().split("?")[0] # Remove query params - inferred_type: Final = extension_to_mime.get(extension) - if inferred_type: - return inferred_type - - # Try to detect from binary content signature (magic bytes) - if content: - detected_type: Final = get_image_type(content[:100]) - if detected_type: - type_to_mime: Final = { - "png": "image/png", - "jpeg": "image/jpeg", - "gif": "image/gif", - "webp": "image/webp", - "heic": "image/heic", - } - if detected_type in type_to_mime: - return type_to_mime[detected_type] - - # If all fallbacks failed, raise error - raise ValueError(f"Unable to determine content type from URL: {url}. Response content-type: {current_content_type}") - - def get_tool_call_names(tools: list[ChatCompletionToolParam]) -> list[str]: """ Get tool call names from tools @@ -2214,32 +1073,6 @@ def convert_custom_tool_format_to_responses_shape(format_obj: Mapping[str, objec } -def get_file_ids_from_messages(messages: list[AllMessageValues]) -> list[str]: - """ - Gets file ids from messages - """ - file_ids: Final = [] - for message in messages: - if message.get("role") == "user": - content = message.get("content") - if content: - if isinstance(content, str): - continue - for c in content: - if isinstance(c, dict) and c["type"] == "file": - file_object = cast(ChatCompletionFileObject, c) - file_object_file_field = file_object.get("file") - if not isinstance(file_object_file_field, dict): - # Content block has `type: "file"` but not the - # OpenAI Chat Completions shape. No file_id to - # extract, so skip instead of raising KeyError. - continue - file_id = file_object_file_field.get("file_id") - if file_id: - file_ids.append(file_id) - return file_ids - - def check_is_function_call(logging_obj: "LoggingClass") -> bool: from litellm.litellm_core_utils.prompt_templates.common_utils import ( is_function_call, @@ -2252,195 +1085,6 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool: return False -_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object]) -PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT: Final = ( - "set `drop_params: true` on the request or in litellm_settings (`litellm.drop_params = True` in the SDK)" -) - - -def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: - """A breakpoint the client already wrote on the block wins, TTL included.""" - if marker is None or target.get("prompt_cache_breakpoint") is not None: - return target - marked: Final = {**target, "prompt_cache_breakpoint": marker} - return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key - - -_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) -_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(ChatCompletionPromptCacheBreakpoint) -_AcceptedBreakpointT: Final = TypeVar("_AcceptedBreakpointT", bound=Mapping[str, object]) - - -def _accepted_prompt_cache_breakpoint( - accepted: TypeAdapter[_AcceptedBreakpointT], marker: object -) -> _AcceptedBreakpointT | None: - try: - return accepted.validate_python(marker) - except ValidationError: - return None - - -def prompt_cache_breakpoint_within( - accepted: TypeAdapter[_AcceptedBreakpointT], marker: object -) -> _AcceptedBreakpointT | None: - """The client's marker cut down to the ``accepted`` shape: an unsupported ttl is dropped and the mode kept, - and a marker whose mode the target cannot read is dropped whole.""" - whole: Final = _accepted_prompt_cache_breakpoint(accepted, marker) - if whole is not None: - return whole - fields: Final = _breakpoint_fields_or_none(marker) - without_ttl: Final = ( - None - if fields is None - else _accepted_prompt_cache_breakpoint(accepted, {key: item for key, item in fields.items() if key != "ttl"}) - ) - verbose_logger.debug( - "Dropping malformed prompt_cache_breakpoint %r under drop_params" - if without_ttl is None - else "Dropping the unsupported ttl of prompt_cache_breakpoint %r under drop_params", - marker, - ) - return without_ttl - - -def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: - if marker is None or not drop_params: - return marker - return prompt_cache_breakpoint_within(_PROMPT_CACHE_BREAKPOINT, marker) - - -def prompt_cache_breakpoint_with_readable_mode(marker: object, drop_params: bool) -> object: - """The client's marker as written, or None under drop_params when no target could read its mode, so the - target's own config decides which ttl and keys it keeps.""" - return None if prompt_cache_breakpoint_for_wire(marker, drop_params) is None else marker - - -def whole_prompt_cache_breakpoint_or_none(marker: object) -> PromptCacheBreakpoint | None: - return _accepted_prompt_cache_breakpoint(_PROMPT_CACHE_BREAKPOINT, marker) - - -def chat_completion_prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: - if marker is None or not drop_params: - return marker - return prompt_cache_breakpoint_within(_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT, marker) - - -def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: - """Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block.""" - if block.get("cache_control") is None: - return block - unmarked: Final = {key: value for key, value in block.items() if key != "cache_control"} - return cast( # cast-ok: same block shape as the input with the marker key renamed - _MarkedT, with_prompt_cache_breakpoint(unmarked, PromptCacheBreakpoint(mode="explicit")) - ) - - -def prompt_cache_breakpoint_as_cache_control(block: _MarkedT, drop_params: bool) -> _MarkedT: - """OpenAI's ``prompt_cache_breakpoint`` mark rewritten as Anthropic's ``cache_control`` on the same block. - A ``cache_control`` the client already wrote on the block wins, TTL included.""" - marker: Final = block.get("prompt_cache_breakpoint") - if marker is None: - return block - unmarked: Final = {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"} - control: Final = ( - None - if block.get("cache_control") is not None - else _cache_control_for_prompt_cache_breakpoint(marker, drop_params) - ) - marked: Final = unmarked if control is None else {**unmarked, "cache_control": control} - return cast(_MarkedT, marked) # cast-ok: same block shape as the input with the marker key renamed - - -def _cache_control_for_prompt_cache_breakpoint(marker: object, drop_params: bool) -> ChatCompletionCachedContent | None: - try: - breakpoint_mark: Final = _PROMPT_CACHE_BREAKPOINT.validate_python(marker) - except ValidationError: - if not drop_params: - raise UnsupportedParamsError( - message=( - f"prompt_cache_breakpoint must be {{'mode': 'explicit'}} with an optional ttl of '30m' or '12h', got " - f"{marker!r}. Fix the marker, or " - f"{PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it." - ), - status_code=400, - ) - verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker) - return None - ttl: Final = breakpoint_mark.get("ttl") - if ttl is not None and not drop_params: - raise UnsupportedParamsError( - message=( - f"This model's prompt cache lasts 5m, or 1h where it offers one, so prompt_cache_breakpoint " - f"ttl={ttl!r} has no equivalent. Remove the ttl, {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to send " - "a 5m breakpoint instead, or send cache_control {'type': 'ephemeral', 'ttl': '1h'} on the block " - "where the model offers a 1h cache." - ), - status_code=400, - ) - return ChatCompletionCachedContent(type="ephemeral") - - -_BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"}) -_BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object]) -_BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object]) - - -def _breakpoint_fields_or_none(value: object) -> Mapping[str, object] | None: - try: - return _BREAKPOINT_FIELDS.validate_python(value) - except ValidationError: - return None - - -def _without_prompt_cache_breakpoint(value: object) -> object: - if not isinstance(value, Mapping): - return value - fields: Final = _BREAKPOINT_FIELDS.validate_python(value) - return {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"} - - -def _breakpoints_stripped_from(key: str, value: object) -> object: - if key in _BREAKPOINT_BLOCK_LIST_KEYS and isinstance(value, list): - return [_without_prompt_cache_breakpoint(block) for block in _BREAKPOINT_BLOCKS.validate_python(value)] - return _without_prompt_cache_breakpoint(value) if key == "function" else value - - -def without_prompt_cache_breakpoints(block: _MarkedT) -> _MarkedT: - """A message or tool definition with every ``prompt_cache_breakpoint`` removed: its own, its content parts', - its tool calls' and its ``function``'s.""" - stripped: Final = { - key: _breakpoints_stripped_from(key, value) for key, value in block.items() if key != "prompt_cache_breakpoint" - } - return cast(_MarkedT, stripped) # cast-ok: same block shape as the input minus the marker keys - - -def _part_with_chat_completion_prompt_cache_breakpoint(part: object) -> object: - if not isinstance(part, Mapping): - return part - fields: Final = _BREAKPOINT_FIELDS.validate_python(part) - marker: Final = fields.get("prompt_cache_breakpoint") - if marker is None: - return fields - unmarked: Final = {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"} - return with_prompt_cache_breakpoint(unmarked, chat_completion_prompt_cache_breakpoint_for_wire(marker, True)) - - -def chat_completion_prompt_cache_breakpoints_for_wire(message: _MarkedT, drop_params: bool) -> _MarkedT: - content: Final = message.get("content") - if not drop_params or not isinstance(content, list): - return message - parts: Final = [ - _part_with_chat_completion_prompt_cache_breakpoint(part) for part in _BREAKPOINT_BLOCKS.validate_python(content) - ] - return cast(_MarkedT, {**message, "content": parts}) # cast-ok: same message shape with its parts' markers trimmed - - -LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS: Final = frozenset( - {"thinking_blocks", "reasoning_items", "provider_specific_fields"} -) -LITELLM_INTERNAL_MESSAGE_FIELDS: Final = LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS | {"reasoning_content"} - - def _tool_call_without_provider_specific_fields(tool_call: object) -> object: if not isinstance(tool_call, dict): return tool_call @@ -2692,267 +1336,6 @@ def convert_prefix_message_to_non_prefix_messages( return new_messages -def _provider_text_or_none(value: object) -> str | None: - return cast(str | None, value) # cast-ok: reasoning fields arrive in untyped provider messages - - -def extract_reasoning_content(message: Mapping[str, object]) -> tuple[str | None, str | None]: - """ - Extract reasoning content and main content from a message. - - Args: - message (dict): The message dictionary that may contain reasoning_content - - Returns: - tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content) - """ - message_content: Final = message.get("content") - if "reasoning_content" in message: - return _provider_text_or_none(message["reasoning_content"]), _provider_text_or_none(message_content) - elif "reasoning" in message: - return _provider_text_or_none(message["reasoning"]), _provider_text_or_none(message_content) - elif isinstance(message_content, str): - return parse_content_for_reasoning(message_content) - return None, _provider_text_or_none(message_content) - - -_extract_reasoning_content = extract_reasoning_content - - -def _readable_thinking_text(block: Mapping[str, object]) -> str: - """The text a chat model can read back, empty for redacted blocks and malformed ones.""" - if block.get("type") != "thinking": - return "" - return str(block.get("thinking") or "") - - -def reasoning_content_from_thinking_blocks( - thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock], -) -> str: - """Flatten Anthropic thinking blocks into the `reasoning_content` string chat models expect. - - Redacted blocks carry no readable text, so they contribute nothing. - """ - return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block))) - - -ENCRYPTED_REASONING_SIGNATURE_PREFIX: Final = "litellm_encrypted_reasoning:" - - -def encrypted_reasoning_signature(encrypted_content: str) -> str: - """The opaque value a Responses API reasoning item's `encrypted_content` travels in. - - Anthropic clients echo a thinking block's `signature` and a redacted block's `data` - back verbatim, so either field can carry the encrypted reasoning across turns; the - prefix tells the two apart from a signature Anthropic minted. - """ - return f"{ENCRYPTED_REASONING_SIGNATURE_PREFIX}{encrypted_content}" - - -def _carries_encrypted_reasoning(signature: object) -> bool: - return isinstance(signature, str) and signature.startswith(ENCRYPTED_REASONING_SIGNATURE_PREFIX) - - -def encrypted_content_from_signature(signature: object) -> str | None: - if not isinstance(signature, str) or not _carries_encrypted_reasoning(signature): - return None - return signature.removeprefix(ENCRYPTED_REASONING_SIGNATURE_PREFIX) or None - - -def _encrypted_reasoning_field(block: Mapping[str, object]) -> object: - match block.get("type"): - case "thinking": - return block.get("signature") - case "redacted_thinking": - return block.get("data") - case _: - return None - - -def encrypted_content_of_block(block: Mapping[str, object]) -> str | None: - return encrypted_content_from_signature(_encrypted_reasoning_field(block)) - - -def is_encrypted_reasoning_block(block: object) -> bool: - """A thinking or redacted_thinking block carrying Responses API encrypted reasoning. - - Only the Responses API that minted the content can read it back, so an Anthropic - backend has to drop such a block rather than fail signature verification on it. - """ - if not isinstance(block, Mapping): - return False - mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance - return _carries_encrypted_reasoning(_encrypted_reasoning_field(mapping)) - - -def is_unsignable_thinking_block(block: object) -> bool: - """A thinking block Anthropic cannot accept on input. - - Anthropic verifies the signature cryptographically, so a block with a null, - empty, or missing signature (e.g. from an open-source reasoning model) is - rejected with a 400, and so is a block whose signature or data carries - another provider's encrypted reasoning. A signed block is kept whatever its - text, since ``display: omitted`` returns signed blocks with empty ``thinking``. - `redacted_thinking` blocks carry no signature and are always kept. - """ - if is_encrypted_reasoning_block(block): - return True - if not isinstance(block, Mapping): - return False - mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance - if mapping.get("type") != "thinking": - return False - signature: Final = mapping.get("signature") - return not (isinstance(signature, str) and len(signature) > 0) - - -def has_blank_redacted_data(block: object) -> bool: - match block: - case {"type": "redacted_thinking", "data": str() as data}: - return not data.strip() - case {"type": "redacted_thinking"}: - return True - case _: - return False - - -def native_redacted_thinking_data(data: str) -> str: - """Claude's own `data` string for a redacted_thinking block. - - Bedrock Converse carries it base64-wrapped as `redactedContent`, and LiteLLM passed that - wrapper through as `data` until it started decoding Converse replies, so histories saved - before then still hold it. Claude's own value never decodes twice, since its first byte - decodes to a protobuf tag outside the base64 alphabet. - """ - return _converse_wrapped_payload(data) or data - - -def converse_redacted_content(data: str) -> str: - """The `redactedContent` Bedrock Converse expects for a redacted_thinking block's `data`.""" - return base64.b64encode(native_redacted_thinking_data(data).encode()).decode() - - -def _converse_wrapped_payload(data: str) -> str | None: - try: - payload: Final = base64.b64decode(data, validate=True).decode("ascii") - _ = base64.b64decode(payload, validate=True) - except (binascii.Error, ValueError): - return None - else: - return payload - - -def strip_encrypted_reasoning_from_messages( - messages: object, - *, - should_strip: Callable[[Mapping[str, object]], bool] | None = None, -) -> None: - """Drop the bridge-tagged reasoning blocks a routed deployment cannot decrypt from - Anthropic-shaped history. - - The whole block goes, the way #40280 drops undecryptable Responses ``input`` items: a - provider that did not mint the block rejects it signed (a foreign signature) and unsigned - (a missing signature) alike, so keeping its text as an unsigned thinking block only moves - the 400 from the router to the provider. - - Mutates the content lists in place: the router's fallback snapshot shares these - message objects, so a rebound list would replay the stripped blocks on the fallback hop. - """ - if not isinstance(messages, list): - return - for content in anthropic_content_lists(cast(list[object], messages)): # cast-ok: untyped client json - _strip_encrypted_reasoning_from_blocks(content, should_strip=should_strip) - - -def anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]: - return ( - cast(list[object], content) # cast-ok: narrowed by isinstance - for message in messages - if isinstance(message, Mapping) - for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance - if isinstance(content, list) - ) - - -def _strip_encrypted_reasoning_from_blocks( - content: object, - *, - should_strip: Callable[[Mapping[str, object]], bool] | None = None, -) -> None: - blocks: Final = cast(list[object], content) # cast-ok: narrowed by the caller's isinstance - kept: Final = tuple( - block - for block in blocks - if not is_encrypted_reasoning_block(block) - or (should_strip is not None and not should_strip(cast(Mapping[str, object], block))) - ) - blocks[:] = kept - - -def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: - index, block = indexed_block - return f"encrypted:{index}" if is_encrypted_reasoning_block(block) else "summary" - - -def _reasoning_item_from_block_group(group: tuple[Mapping[str, object], ...]) -> ChatCompletionReasoningItem | None: - summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [ # mutable-ok: API message payload - ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text) - for block in group - if (text := _readable_thinking_text(block)) - ] - encrypted_content: Final = encrypted_content_of_block(group[0]) - if encrypted_content is not None: - return ChatCompletionReasoningItem(type="reasoning", summary=summary, encrypted_content=encrypted_content) - if not summary: - return None - return ChatCompletionReasoningItem(type="reasoning", summary=summary) - - -def responses_reasoning_items_from_thinking_blocks( - thinking_blocks: Iterable[Mapping[str, object]], -) -> tuple[ChatCompletionReasoningItem, ...]: - """Build Responses API `reasoning` input items from Anthropic thinking blocks. - - A block carrying encrypted reasoning replays the item it came from byte for byte; - a run of plain thinking blocks collapses into one summary-only item. No item carries - an `id`: the Responses API 404s on any id it did not mint itself and rejects an empty - one, while an item without an id is always accepted. - """ - return tuple( - item - for _, group in groupby(enumerate(thinking_blocks), key=_reasoning_replay_group_key) - if (item := _reasoning_item_from_block_group(tuple(block for _, block in group))) is not None - ) - - -def parse_content_for_reasoning( - message_text: str | None, -) -> tuple[str | None, str | None]: - """ - Parse the content for reasoning - - Returns: - - reasoning_content: The content of the reasoning - - content: The content of the message - """ - if not message_text: - return None, message_text - - reasoning_match: Final = re.match( - r"<(?:think|thinking|budget:thinking)>(.*?)(.*)", - message_text, - re.DOTALL, - ) - - if reasoning_match: - return reasoning_match.group(1), reasoning_match.group(2) - - return None, message_text - - -_parse_content_for_reasoning = parse_content_for_reasoning - - def _extract_base64_data(image_url: str) -> str: """ Extract pure base64 data from an image URL. @@ -3163,252 +1546,6 @@ def merge_consecutive_system_messages( ] -_DECODED_JSON: Final = TypeAdapter(object) - - -def _attempt_json_repair(s: str) -> object | None: - """ - Attempt to repair truncated JSON produced by LLM tool calls. - - Handles the most common truncation patterns where the model generates - valid JSON that is cut short (missing closing brackets/braces). - - Returns the parsed value on success, or None if repair fails. - """ - import json - - stripped: Final = s.rstrip() - if not stripped: - return None - - # Track the stack of unmatched openers to respect nesting order - opener_stack: Final[list] = [] - in_string = False - escape_next = False - - for ch in stripped: - if escape_next: - escape_next = False - continue - if ch == "\\": - if in_string: - escape_next = True - continue - if ch == '"': - in_string = not in_string - continue - if in_string: - continue - if ch == "{": - opener_stack.append("}") - elif ch == "[": - opener_stack.append("]") - elif ch in ("}", "]"): - if opener_stack and opener_stack[-1] == ch: - opener_stack.pop() - - if not opener_stack: - return None - - # Remove trailing comma before we close brackets - candidate = stripped.rstrip(",") - - # Close in reverse order of opening (respects nesting) - candidate += "".join(reversed(opener_stack)) - - try: - return _DECODED_JSON.validate_python(json.loads(candidate)) - except json.JSONDecodeError: - pass - - return None - - -def parse_tool_call_arguments( - arguments: str | None, - tool_name: str | None = None, - context: str | None = None, -) -> Any: - """ - Parse tool call arguments from a JSON string. - - When the JSON is malformed (e.g. truncated by the model), this function - attempts a lightweight repair (closing unmatched brackets/braces) before - raising an error. A warning is logged whenever repair succeeds so that - callers are aware the arguments were not perfectly formed. - - Args: - arguments: The JSON string containing tool arguments, or None. - tool_name: Optional name of the tool (for error messages). - context: Optional context string (e.g., "Anthropic Messages API"). - - Returns: - Parsed arguments (usually a dict, but may be any JSON-deserializable - type such as list, str, int, float, or None). Returns empty dict if - arguments is None or empty. - - Raises: - ValueError: If the arguments string is not valid JSON and cannot be repaired. - """ - import json - - if not arguments or not arguments.strip(): - return {} - - try: - return _DECODED_JSON.validate_python(json.loads(arguments)) - except json.JSONDecodeError as original_error: - repaired: Final = _attempt_json_repair(arguments) - if repaired is not None: - verbose_logger.warning( - "Repaired truncated tool call arguments for tool '%s' (%s). Original (%d chars): %.200s%s", - tool_name or "", - context or "unknown context", - len(arguments), - arguments, - "..." if len(arguments) > 200 else "", - ) - return repaired - - error_parts: Final = ["Failed to parse tool call arguments"] - - if tool_name: - error_parts.append(f"for tool '{tool_name}'") - if context: - error_parts.append(f"({context})") - - error_message: Final = " ".join(error_parts) + f". Error: {original_error}. Arguments: {arguments}" - - raise ValueError(error_message) from original_error - - -def split_concatenated_json_objects(raw: str) -> list[dict[str, object]]: - """ - Split a string that contains one or more concatenated JSON objects into - a list of parsed dicts. - - LLM providers (notably Bedrock Claude Sonnet 4.5) sometimes return - multiple tool-call argument objects concatenated in a single - ``arguments`` string, e.g.:: - - '{"command":["curl",...]}{"command":["curl",...]}{"command":["curl",...]}' - - ``json.loads()`` fails on this with ``JSONDecodeError: Extra data``. - This helper uses ``json.JSONDecoder.raw_decode()`` to walk the string - and extract each JSON object individually. - - The walk degrades gracefully: if the string is malformed or truncated - (e.g. a stream that ended mid-tool-call), whatever complete objects were - parsed before the bad tail are returned and the remainder is discarded - with a warning, rather than raising. The sole caller - (``_convert_to_bedrock_tool_call_invoke``) treats an empty result as - ``input={}`` so the conversation can continue instead of hard-failing. - - Returns - ------- - list[dict] - A list of parsed dicts – one per JSON object found. If *raw* is - empty, whitespace-only, or wholly unparseable, an empty list is - returned. - """ - import json - - raw = raw.strip() - if not raw: - return [] - - decoder: Final = json.JSONDecoder() - results: Final[list[dict[str, object]]] = [] - idx = 0 - length: Final = len(raw) - - while idx < length: - # Skip whitespace between objects - while idx < length and raw[idx] in " \t\n\r": - idx += 1 - if idx >= length: - break - - try: - decoded = decoder.raw_decode(raw, idx) - except json.JSONDecodeError as e: - verbose_logger.warning( - "split_concatenated_json_objects: discarding unparseable tool-call " - "arguments tail after %d complete object(s); decode_start=%d error=%s", - len(results), - idx, - e, - ) - break - obj = as_str_mapping(_DECODED_JSON.validate_python(decoded[0])) - if obj is not None: - results.append(dict(obj)) - else: - # Non-dict JSON value – wrap in empty dict (Bedrock requires - # toolUse.input to be an object). - results.append({}) - idx = decoded[1] - - return results - - -MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS: Final = 8 - - -def salvage_concatenated_tool_arguments(raw: str) -> tuple[dict[str, object], ...]: - """Return complete concatenated JSON objects that are safe to expand. - - Identical objects collapse to the first one and are not capped. More than - ``MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS`` objects that are not all identical - returns an empty tuple. Anything that is not a full concatenation of JSON - objects returns an empty tuple. Repeated copies of the first object are not - retained, and once the cap is passed the rest of the string is only checked. - """ - stripped: Final = raw.strip() - if not stripped: - return () - decoder: Final = json.JSONDecoder() - length: Final = len(stripped) - idx = 0 # rebind-ok: cursor walks the concatenated JSON string - count = 0 # rebind-ok: counts complete objects without retaining duplicates - kept = () # rebind-ok: holds at most one object past the salvage cap - exceeded = False # rebind-ok: cap already passed, the tail is only validated - while idx < length: - while idx < length and stripped[idx] in " \t\n\r": - idx += 1 - if idx >= length: - break - try: - decoded = decoder.raw_decode(stripped, idx) - except json.JSONDecodeError: - return () - decoded_object = as_str_mapping(_DECODED_JSON.validate_python(decoded[0])) - if decoded_object is None: - return () - idx = decoded[1] - if exceeded: - continue - obj = dict(decoded_object) - count += 1 - if not kept: - kept = (obj,) - continue - if obj == kept[0] and len(kept) == 1: - continue - if len(kept) == 1 and count > 2 and count - 1 > MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS: - exceeded = True - continue - if len(kept) == 1 and count > 2: - kept = (kept[0],) * (count - 1) - if len(kept) >= MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS: - exceeded = True - continue - kept = (*kept, obj) - if exceeded: - return () - return kept - - def text_completion_prompt_to_messages(prompt: object) -> tuple[AllMessageValues, ...]: """ Wrap an OpenAI ``/v1/completions`` ``prompt`` into Chat Completion messages. diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index eda98ae2c67..c6288227546 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -4291,7 +4291,7 @@ def _convert_to_bedrock_tool_call_invoke( - extract name - extract id """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( split_concatenated_json_objects, ) diff --git a/litellm/litellm_core_utils/prompt_templates/file_id_utils.py b/litellm/litellm_core_utils/prompt_templates/file_id_utils.py new file mode 100644 index 00000000000..db74ccf1c6d --- /dev/null +++ b/litellm/litellm_core_utils/prompt_templates/file_id_utils.py @@ -0,0 +1,635 @@ +import io +import mimetypes +import re +from collections.abc import Mapping +from os import PathLike +from pathlib import Path +from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move + +from litellm import verbose_logger +from litellm.litellm_core_utils.batch_utils import InMemoryFile +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionFileObject, +) +from litellm.types.utils import ( + ExtractedFileData, + FileTypes, + SpecialEnums, +) + + +def get_format_from_file_id(file_id: str | None) -> str | None: + """ + Gets format from file id + + unified_file_id = litellm_proxy:{};unified_id,{} + If not a unified file id, returns 'file' as default format + """ + from litellm.proxy.openai_files_endpoints.common_utils import ( + convert_b64_uid_to_unified_uid, + ) + + if not file_id: + return None + try: + transformed_file_id: Final = convert_b64_uid_to_unified_uid(file_id) + if transformed_file_id.startswith(SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value): + match: Final = re.match( + f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}:(.*?);unified_id", + transformed_file_id, + ) + if match: + return match.group(1) + + return None + except Exception: + return None + + +def update_messages_with_model_file_ids( + messages: list[AllMessageValues], + model_id: str | None, + model_file_id_mapping: dict[str, dict[str, str]], +) -> list[AllMessageValues]: + """ + Updates messages with model file ids. + + For managed files (unified file IDs), uses model_file_id_mapping if it + resolves the id, otherwise decodes the base64-encoded unified file ID + and extracts the llm_output_file_id directly. Mirrors the Responses-API + sibling `update_responses_input_with_model_file_ids`. + + model_file_id_mapping: Dict[str, Dict[str, str]] = { + "litellm_proxy/file_id": { + "model_id": "provider_file_id" + } + } + """ + from litellm.proxy.openai_files_endpoints.common_utils import ( + convert_b64_uid_to_unified_uid, + get_original_file_id, + is_base64_encoded_unified_file_id, + is_model_embedded_id, + ) + + for message in messages: + if message.get("role") == "user": + content = message.get("content") + if content: + if isinstance(content, str): + continue + for c in content: + if not isinstance(c, dict): + # Content list items aren't always dicts. e.g. + # text_completion forwards a token-ids list/list-of- + # lists through this path. Skip non-dict items + # instead of indexing into them. + continue + if c.get("type") == "file": + file_object = cast(ChatCompletionFileObject, c) + file_object_file_field = file_object.get("file") + if not isinstance(file_object_file_field, dict): + # Content block has `type: "file"` but not the + # OpenAI Chat Completions shape (e.g. a LangChain + # v1 standardized file block, or a provider-native + # shape that also uses `type: "file"`). Nothing to + # remap here, so skip instead of crashing. + continue + file_id = file_object_file_field.get("file_id") + format = file_object_file_field.get("format", get_format_from_file_id(file_id)) + + if file_id: + provider_file_id = ( + model_file_id_mapping.get(file_id, {}).get(model_id) + if model_file_id_mapping and model_id is not None + else None + ) + if not provider_file_id and is_base64_encoded_unified_file_id(file_id): + unified_file_id = convert_b64_uid_to_unified_uid(file_id) + if "llm_output_file_id," in unified_file_id: + provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0] + if not provider_file_id and is_model_embedded_id(file_id): + provider_file_id = get_original_file_id(file_id) + file_object_file_field["file_id"] = provider_file_id or file_id + if format: + file_object_file_field["format"] = format + return messages + + +def update_responses_input_with_model_file_ids( + input: object, + model_id: str | None = None, + model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None, +) -> object: + """ + Updates responses API input with provider-specific file IDs. + File IDs are always inside the content array, not as direct input_file items. + + For managed files (unified file IDs), uses model_file_id_mapping if provided, + otherwise decodes the base64-encoded unified file ID and extracts the llm_output_file_id directly. + + Args: + input: The responses API input parameter + model_id: The model ID to use for looking up provider-specific file IDs + model_file_id_mapping: Dictionary mapping litellm file IDs to provider file IDs + Format: {"litellm_file_id": {"model_id": "provider_file_id"}} + """ + from litellm.proxy.openai_files_endpoints.common_utils import ( + convert_b64_uid_to_unified_uid, + get_original_file_id, + is_base64_encoded_unified_file_id, + is_model_embedded_id, + ) + + if isinstance(input, str): + return input + + if not isinstance(input, list): + return input + + updated_input: Final = [] + for item in input: + if not isinstance(item, dict): + updated_input.append(item) + continue + + updated_item = item.copy() + content = item.get("content") + if isinstance(content, list): + updated_content = [] + for content_item in content: + if isinstance(content_item, dict) and content_item.get("type") == "input_file": + file_id = content_item.get("file_id") + if isinstance(file_id, str) and file_id: + provider_file_id = file_id # Default to original + + # Check if we have a mapping for this file ID + if model_file_id_mapping and model_id and file_id in model_file_id_mapping: + # Use the model-specific file ID from mapping + provider_file_id = model_file_id_mapping.get(file_id, {}).get(model_id) or file_id + updated_content_item = content_item.copy() + updated_content_item["file_id"] = provider_file_id + updated_content.append(updated_content_item) + else: + # Check if this is a base64-encoded unified file ID without mapping + is_unified_file_id = is_base64_encoded_unified_file_id(file_id) + if is_unified_file_id: + # Fallback: decode unified file ID + unified_file_id = convert_b64_uid_to_unified_uid(file_id) + if "llm_output_file_id," in unified_file_id: + provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0] + + updated_content_item = content_item.copy() + updated_content_item["file_id"] = provider_file_id + updated_content.append(updated_content_item) + elif is_model_embedded_id(file_id): + updated_content_item = content_item.copy() + updated_content_item["file_id"] = get_original_file_id(file_id) + updated_content.append(updated_content_item) + else: + # Not a managed file, keep as-is + updated_content.append(content_item) + else: + updated_content.append(content_item) + else: + updated_content.append(content_item) + updated_item["content"] = updated_content + + updated_input.append(updated_item) + + return updated_input + + +def _decode_vector_store_ids_in_tools( + tools: list[dict[str, object]] | None, +) -> list[dict[str, object]] | None: + """ + Decodes unified (LiteLLM-managed) vector_store_ids in file_search tools to + provider-native IDs. Non-unified IDs are passed through unchanged. + + This runs unconditionally — no file-ID mapping is required. + """ + if not tools or not isinstance(tools, list): + return tools + + from litellm.llms.base_llm.managed_resources.utils import ( + is_base64_encoded_unified_id, + parse_unified_id, + ) + + updated_tools: Final = [] + for tool in tools: + if not isinstance(tool, dict) or tool.get("type") != "file_search": + updated_tools.append(tool) + continue + + vector_store_ids = tool.get("vector_store_ids") + if not isinstance(vector_store_ids, list): + updated_tools.append(tool) + continue + + decoded_ids = [] + for vs_id in vector_store_ids: + if not isinstance(vs_id, str) or not is_base64_encoded_unified_id(vs_id): + decoded_ids.append(vs_id) + continue + + parsed = parse_unified_id(vs_id) + provider_resource_id = parsed.get("provider_resource_id") if parsed else None + + if not provider_resource_id: + verbose_logger.warning( + "file_search tool contains unified vector_store_id '%s' that could " + "not be decoded to a provider resource ID — passing original ID. " + "Ensure the vector store was created via LiteLLM.", + vs_id, + ) + decoded_ids.append(vs_id) + else: + decoded_ids.append(provider_resource_id) + + updated_tools.append({**tool, "vector_store_ids": decoded_ids}) + + return updated_tools + + +def update_responses_tools_with_model_file_ids( + tools: list[dict[str, object]] | None, + model_id: str | None = None, + model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None, +) -> list[dict[str, object]] | None: + """ + Updates responses API tools with provider-specific file IDs. + + Pass 1 (always): decode unified vector_store_ids in file_search tools. + Pass 2 (needs mapping): map code_interpreter container file_ids to provider IDs. + + Args: + tools: The responses API tools parameter + model_id: The model ID to use for looking up provider-specific file IDs + model_file_id_mapping: Dictionary mapping litellm file IDs to provider file IDs + Format: {"litellm_file_id": {"model_id": "provider_file_id"}} + """ + if not tools or not isinstance(tools, list): + return tools + + # Pass 1: decode unified vector_store_ids (no mapping needed) + tools = _decode_vector_store_ids_in_tools(tools) or tools + + # Pass 2: map code_interpreter file IDs (requires mapping) + if not model_file_id_mapping or not model_id: + return tools + + updated_tools: Final = [] + for tool in tools: + if not isinstance(tool, dict): + updated_tools.append(tool) + continue + + updated_tool = tool.copy() + + # Handle code_interpreter with container file_ids + if tool.get("type") == "code_interpreter": + container = tool.get("container") + if isinstance(container, dict): + container_file_ids = container.get("file_ids") + if isinstance(container_file_ids, list): + updated_file_ids = [] + for file_id in container_file_ids: + if isinstance(file_id, str): + # Check if we have a mapping for this file ID + if file_id in model_file_id_mapping: + # Map to provider-specific file ID + provider_file_id = model_file_id_mapping.get(file_id, {}).get(model_id) or file_id + updated_file_ids.append(provider_file_id) + else: + updated_file_ids.append(file_id) + else: + updated_file_ids.append(file_id) + + # Update the tool with new file IDs + updated_container = container.copy() + updated_container["file_ids"] = updated_file_ids + updated_tool["container"] = updated_container + + updated_tools.append(updated_tool) + + return updated_tools + + +def extract_file_metadata(file_data: FileTypes) -> tuple[str | None, str | None]: + """ + Resolve (filename, content_type) without reading the file body. + + Mirrors extract_file_data's metadata resolution but never calls .read(), so + it stays O(1) on large uploads. Use this when only metadata is needed (batch + detection, GCS object naming) and the body must remain a streamable Path/handle. + """ + filename: str | None = None + content_type: str | None = None + file_content: object = None + + if isinstance(file_data, tuple): + if len(file_data) == 2: + filename, file_content = file_data + elif len(file_data) == 3: + filename, file_content, content_type = file_data + elif len(file_data) == 4: + filename, file_content, content_type, _ = file_data + elif isinstance(file_data, InMemoryFile): + filename = file_data.name + content_type = file_data.content_type + else: + file_content = file_data + + if filename is None: + if isinstance(file_content, PathLike): + filename = Path(file_content).name + elif isinstance(file_content, io.IOBase): + name_attr: Final = getattr(file_content, "name", None) + if isinstance(name_attr, str): + filename = Path(name_attr).name + + if not content_type: + guessed: Final = mimetypes.guess_type(filename)[0] if filename else None + content_type = guessed or "application/octet-stream" + + return filename, content_type + + +def extract_file_data(file_data: FileTypes) -> ExtractedFileData: + """ + Extracts and processes file data from various input formats. + + Args: + file_data: Can be a tuple of (filename, content, [content_type], [headers]) or direct file content + + Returns: + ExtractedFileData containing: + - filename: Name of the file if provided + - content: The file content in bytes + - content_type: MIME type of the file + - headers: Any additional headers + """ + # Parse the file_data based on its type + filename = None + file_content = None + content_type = None + file_headers: Mapping[str, str] = {} + + if isinstance(file_data, tuple): + if len(file_data) == 2: + filename, file_content = file_data + elif len(file_data) == 3: + filename, file_content, content_type = file_data + elif len(file_data) == 4: + filename, file_content, content_type, file_headers = file_data + elif isinstance(file_data, InMemoryFile): + filename = file_data.name + file_content = file_data + content_type = file_data.content_type + else: + file_content = file_data + # Convert content to bytes + if isinstance(file_content, str): + # Bare string inputs are rejected: when this helper runs in a proxy + # request handler the string came from an attacker-controlled form + # field, and opening it as a path is an arbitrary file read on the + # proxy host. SDK callers who want to upload from a path should + # either pass a pathlib.Path (a PathLike instance — see the branch + # below) or open the file themselves and pass the handle / bytes. + raise ValueError( + "extract_file_data does not accept bare str inputs. Pass bytes, " + "an open file handle, a (filename, content) tuple, or a " + "pathlib.Path. To upload a local file from a path, call " + "open(path, 'rb') yourself." + ) + if isinstance(file_content, PathLike): + # PathLike (pathlib.Path) is a Python-level type that HTTP form + # values can't fabricate. Treat as a local file path for SDK + # convenience. + if filename is None: + filename = Path(file_content).name + with open(file_content, "rb") as f: + content = f.read() + elif isinstance(file_content, io.IOBase): + # If it's a file-like object + # Try to get filename from file handle if not already set + if not filename and hasattr(file_content, "name"): + filename = Path(file_content.name).name + + content = file_content.read() + + if isinstance(content, str): + content = content.encode("utf-8") + # Reset file pointer to beginning + file_content.seek(0) + elif isinstance(file_content, bytes): + content = file_content + else: + raise ValueError(f"Unsupported file content type: {type(file_content)}") + + # Use provided content type or guess based on filename + if not content_type: + if filename: + guessed_type: Final = mimetypes.guess_type(filename)[0] + content_type = guessed_type if guessed_type else "application/octet-stream" + else: + content_type = "application/octet-stream" + + return ExtractedFileData( + filename=filename, + content=content, + content_type=content_type, + headers=file_headers, + ) + + +def get_image_mime_type_from_url(url: str) -> str | None: + """ + Get mime type for common image URLs + See gemini mime types: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/image-understanding#image-requirements + + Supported by Gemini: + application/pdf + audio/mpeg + audio/mp3 + audio/wav + audio/ogg + image/png + image/jpeg + image/webp + text/plain + video/mov + video/mpeg + video/mp4 + video/mpg + video/avi + video/wmv + video/mpegps + video/flv + """ + from urllib.parse import urlparse + + url = url.lower() + + # Parse URL to extract path without query parameters + # This handles URLs like: https://example.com/image.jpg?signature=... + parsed: Final = urlparse(url) + path: Final = parsed.path + + # Map file extensions to mime types + mime_types: Final = { + # Images + (".jpg", ".jpeg"): "image/jpeg", + (".png",): "image/png", + (".webp",): "image/webp", + # Videos + (".mp4",): "video/mp4", + (".mov",): "video/mov", + (".mpeg", ".mpg"): "video/mpeg", + (".avi",): "video/avi", + (".wmv",): "video/wmv", + (".mpegps",): "video/mpegps", + (".flv",): "video/flv", + # Audio + (".mp3",): "audio/mp3", + (".wav",): "audio/wav", + (".mpeg",): "audio/mpeg", + (".ogg",): "audio/ogg", + # Documents + (".pdf",): "application/pdf", + (".txt",): "text/plain", + } + + # Check each extension group against the URL + for extensions, mime_type in mime_types.items(): + if any(path.endswith(ext) for ext in extensions): + return mime_type + + return None + + +_get_image_mime_type_from_url = get_image_mime_type_from_url + + +def infer_content_type_from_url_and_content( + url: str, + content: bytes, + current_content_type: str | None = None, +) -> str: + """ + Infer content type from URL extension and binary content when content-type header is missing or generic. + + This helper implements a fallback strategy for determining MIME types when HTTP headers + are missing or provide generic values (like binary/octet-stream). It's commonly used + when processing images and documents from various sources (S3, URLs, etc.). + + Fallback Strategy: + 1. If current_content_type is valid (not None and not generic octet-stream), return it + 2. Try to infer from URL extension (handles query parameters) + 3. Try to detect from binary content signature (magic bytes) + 4. Raise ValueError if all methods fail + + Args: + url: The URL of the content (used to extract file extension) + content: The binary content (first ~100 bytes are sufficient for detection) + current_content_type: The current content-type from headers (may be None or generic) + + Returns: + str: The inferred MIME type (e.g., "image/png", "application/pdf") + + Raises: + ValueError: If content type cannot be determined by any method + + Example: + >>> content_type = infer_content_type_from_url_and_content( + ... url="https://s3.amazonaws.com/bucket/image.png?AWSAccessKeyId=123", + ... content=png_binary_data, + ... current_content_type="binary/octet-stream" + ... ) + >>> print(content_type) + "image/png" + """ + from litellm.litellm_core_utils.token_counter import get_image_type + + media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None + + # If we have a valid content type that's not generic, use it + if media_type and media_type not in [ + "binary/octet-stream", + "application/octet-stream", + ]: + return media_type + + # Extension to MIME type mapping + # Supports images, documents, and other common file types + extension_to_mime: Final = { + # Image formats + "jpg": "image/jpeg", + "jpeg": "image/jpeg", + "png": "image/png", + "gif": "image/gif", + "webp": "image/webp", + # Document formats + "pdf": "application/pdf", + "csv": "text/csv", + "doc": "application/msword", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "xls": "application/vnd.ms-excel", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "html": "text/html", + "txt": "text/plain", + "md": "text/markdown", + } + + # Try to infer from URL extension + if url: + extension: Final = url.split(".")[-1].lower().split("?")[0] # Remove query params + inferred_type: Final = extension_to_mime.get(extension) + if inferred_type: + return inferred_type + + # Try to detect from binary content signature (magic bytes) + if content: + detected_type: Final = get_image_type(content[:100]) + if detected_type: + type_to_mime: Final = { + "png": "image/png", + "jpeg": "image/jpeg", + "gif": "image/gif", + "webp": "image/webp", + "heic": "image/heic", + } + if detected_type in type_to_mime: + return type_to_mime[detected_type] + + # If all fallbacks failed, raise error + raise ValueError(f"Unable to determine content type from URL: {url}. Response content-type: {current_content_type}") + + +def get_file_ids_from_messages(messages: list[AllMessageValues]) -> list[str]: + """ + Gets file ids from messages + """ + file_ids: Final = [] + for message in messages: + if message.get("role") == "user": + content = message.get("content") + if content: + if isinstance(content, str): + continue + for c in content: + if isinstance(c, dict) and c["type"] == "file": + file_object = cast(ChatCompletionFileObject, c) + file_object_file_field = file_object.get("file") + if not isinstance(file_object_file_field, dict): + # Content block has `type: "file"` but not the + # OpenAI Chat Completions shape. No file_id to + # extract, so skip instead of raising KeyError. + continue + file_id = file_object_file_field.get("file_id") + if file_id: + file_ids.append(file_id) + return file_ids diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index ffc0e82e410..fa6f1c1ed0a 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -21,7 +21,9 @@ import litellm from litellm import verbose_logger from litellm.caching.caching import InMemoryCache from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB, NON_LLM_CONNECTION_TIMEOUT -from litellm.litellm_core_utils.prompt_templates.common_utils import infer_content_type_from_url_and_content +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + infer_content_type_from_url_and_content, +) from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get from litellm.types.llms.openai import AllMessageValues diff --git a/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py b/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py new file mode 100644 index 00000000000..1d6d28c2680 --- /dev/null +++ b/litellm/litellm_core_utils/prompt_templates/json_schema_utils.py @@ -0,0 +1,471 @@ +import re +from collections.abc import Callable, Iterator, Mapping +from graphlib import CycleError, TopologicalSorter +from itertools import chain, islice +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter + +_TOP_LEVEL_SCHEMA_COMBINATORS: Final = ("allOf", "anyOf", "oneOf") +_OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS: Final = ("enum", "const", "not") +_LOCAL_SCHEMA_REF_PREFIXES: Final = (("#/$defs/", "$defs"), ("#/definitions/", "definitions")) +_MAX_SCHEMA_FLATTEN_DEPTH: Final = 32 +_EMPTY_SCHEMA: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _schema_properties(schema: Mapping[str, object]) -> Mapping[str, object]: + properties: Final = schema.get("properties") + return properties if isinstance(properties, dict) else _EMPTY_SCHEMA + + +def _schema_branches(schema: Mapping[str, object], combinator: str) -> tuple[object, ...]: + branches: Final = schema.get(combinator) + return tuple(branches) if isinstance(branches, list) else () + + +def _schema_required_names(schema: Mapping[str, object]) -> frozenset[str]: + required: Final = schema.get("required") + if not isinstance(required, list): + return frozenset() + return frozenset(name for name in required if isinstance(name, str)) + + +def _combinator_required_names(combinator: str, branches: tuple[Mapping[str, object], ...]) -> frozenset[str]: + branch_names: Final = tuple(_schema_required_names(branch) for branch in branches) + if not branch_names: + return frozenset() + if combinator == "allOf": + return branch_names[0].union(*branch_names[1:]) + return branch_names[0].intersection(*branch_names[1:]) + + +def _resolve_local_schema_ref(root: Mapping[str, object], ref: str) -> Mapping[str, object] | None: + matched: Final = next( + ((prefix, container) for prefix, container in _LOCAL_SCHEMA_REF_PREFIXES if ref.startswith(prefix)), + None, + ) + if matched is None: + return None + prefix, container = matched + definitions: Final = root.get(container) + if not isinstance(definitions, dict): + return None + target: Final = definitions.get(ref[len(prefix) :]) + return target if isinstance(target, dict) else None + + +def _mergeable_branch( + root: Mapping[str, object], + branch: object, + seen_refs: frozenset[str], + depth: int, + expanded_refs: dict[str, Mapping[str, object] | None], # mutable-ok: per-call memo bounding repeated $ref work +) -> Mapping[str, object] | None: + if not isinstance(branch, dict) or depth > _MAX_SCHEMA_FLATTEN_DEPTH: + return None + ref: Final = branch.get("$ref") + if not isinstance(ref, str): + flattened: Final = _flatten_schema_against_root(branch, root, seen_refs, depth, expanded_refs) + if any(combinator in flattened for combinator in _TOP_LEVEL_SCHEMA_COMBINATORS): + return None + return flattened + if ref in expanded_refs: + return expanded_refs[ref] + if ref in seen_refs: + return None + target: Final = _resolve_local_schema_ref(root, ref) + expanded: Final = ( + None + if target is None + else _mergeable_branch(root, target, seen_refs | frozenset((ref,)), depth + 1, expanded_refs) + ) + expanded_refs[ref] = expanded + return expanded + + +def _is_object_schema(schema: Mapping[str, object]) -> bool: + return schema.get("type") == "object" or ("type" not in schema and "properties" in schema) + + +def _flatten_schema_against_root( + schema: Mapping[str, object], + root: Mapping[str, object], + seen_refs: frozenset[str], + depth: int, + expanded_refs: dict[str, Mapping[str, object] | None], # mutable-ok: per-call memo bounding repeated $ref work +) -> Mapping[str, object]: + raw_branch_groups: Final = tuple( + ( + combinator, + tuple( + _mergeable_branch(root, branch, seen_refs, depth + 1, expanded_refs) + for branch in _schema_branches(schema, combinator) + ), + ) + for combinator in _TOP_LEVEL_SCHEMA_COMBINATORS + if isinstance(schema.get(combinator), list) + ) + dropped: Final = ( + *(combinator for combinator, _ in raw_branch_groups), + *(key for key in _OPENAI_REJECTED_TOP_LEVEL_SCHEMA_KEYS if key in schema), + ) + if not dropped: + return schema + + if any(branch is None for _, group in raw_branch_groups for branch in group): + return schema + branch_groups: Final = tuple( + (combinator, tuple(branch for branch in group if branch is not None)) for combinator, group in raw_branch_groups + ) + branches: Final = tuple(branch for _, group in branch_groups for branch in group) + is_object_schema: Final = _is_object_schema(schema) or ( + "type" not in schema and branches != () and all(_is_object_schema(branch) for branch in branches) + ) + if not is_object_schema: + return schema + + merged_properties: Final = { + name: value for source in (*reversed(branches), schema) for name, value in _schema_properties(source).items() + } + required_names: Final = _schema_required_names(schema).union( + *(_combinator_required_names(combinator, group) for combinator, group in branch_groups) + ) + kept: Final = MappingProxyType({key: value for key, value in schema.items() if key not in dropped}) + required_update: Final = MappingProxyType({"required": sorted(required_names)}) if required_names else _EMPTY_SCHEMA + return { + **kept, + "type": "object", + "properties": merged_properties, + **required_update, + } + + +def flatten_top_level_schema_combinators(schema: Mapping[str, object]) -> Mapping[str, object]: + """Merge top-level ``allOf``/``anyOf``/``oneOf`` branches into an object tool schema. + + OpenAI's function-calling validator rejects tool ``parameters`` carrying + 'oneOf'/'anyOf'/'allOf'/'enum'/'const'/'not' at the top level (nested uses + are accepted), while lenient backends such as the ChatGPT backend Codex + talks to natively accept them, so an MCP tool declaring a top-level union + 400s through LiteLLM. Branch properties merge without clobbering (the + top-level schema wins, then earlier branches); ``required`` becomes the + top-level list plus the intersection of the branch lists for anyOf/oneOf + or their union for allOf. Branches that are local ``$ref``s + (``#/$defs/...`` or ``#/definitions/...``) are resolved first, each ref + at most once per call, and branches that are themselves combinators are + flattened recursively up to a fixed depth; a branch that cannot be fully + merged (a boolean schema, an external or cyclic ``$ref``, a non-object + union, or nesting past the depth cap) leaves the whole schema untouched so + OpenAI's own validation still applies. Non-object schemas pass through + unchanged and the input is never mutated. + """ + return _flatten_schema_against_root(schema, schema, frozenset(), 0, {}) + + +_SUBSCHEMA_KEYWORDS: Final = frozenset( + { + "additionalItems", + "additionalProperties", + "contains", + "else", + "if", + "items", + "not", + "propertyNames", + "then", + "unevaluatedItems", + "unevaluatedProperties", + } +) +_SUBSCHEMA_LIST_KEYWORDS: Final = frozenset({"allOf", "anyOf", "items", "oneOf", "prefixItems"}) +_SUBSCHEMA_MAP_KEYWORDS: Final = frozenset( + {"$defs", "definitions", "dependentSchemas", "patternProperties", "properties"} +) + +_MAX_SCHEMA_NESTING: Final = 1024 + + +def drop_non_python_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]: + """Drop every regex in a schema position that Python's ``re`` cannot compile. + + OpenAI validates tool ``parameters`` against the 2020-12 metaschema with + ``jsonschema``'s format checker, which hands each ``pattern`` value and each + ``patternProperties`` key to ``re.compile``, so a regex written for an + ECMA-262 engine (Unicode property escapes such as ``\\p{Cc}``, as in Claude + Code's ``Artifact`` tool) is refused with "'...' is not a 'regex'" by every + model family on both the chat and Responses wires. Only schema positions are + walked (properties, items, combinators, ``$defs`` and the other applicators), + so a ``pattern`` key inside ``default``, ``examples``, ``const`` or vendor + extensions is data and stays. Outside strict mode the keyword is only a + hint, so dropping it costs the model a constraint and the caller nothing. + Compilable regexes and everything else pass through, the input is never + mutated, and the same object comes back when nothing was dropped. The walk + is level-order rather than recursive, rebuilt deepest level first, and stops + at more schema levels than a JSON parser admits, so a cyclic schema built in + code cannot spin it. + """ + return _schema_without_rejected_regex(schema, _is_not_python_regex) + + +def drop_lookaround_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]: + """Drop every regex in a schema position that uses a lookaround assertion. + + Some Bedrock Converse families compile tool schema regexes with an engine that + has no lookahead or lookbehind and refuse the whole request over one. The ``(?=``, + ``(?!``, ``(?<=`` and ``(? Mapping[str, object]: + rebuilt: dict[int, Mapping[str, object]] = {} # mutable-ok: per-call memo of rewritten nodes, deepest level first + for level in reversed(tuple(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING))): + rebuilt.update( + (id(node), rewritten) + for node in level + if (rewritten := _node_without_rejected_regex(node, rebuilt, rejected)) is not node + ) + return rebuilt.get(id(schema), schema) + + +def _schema_levels(schema: Mapping[str, object]) -> Iterator[tuple[Mapping[str, object], ...]]: + frontier: tuple[Mapping[str, object], ...] = (schema,) # rebind-ok: level-order cursor, one level a round + while frontier: + yield frontier + frontier = tuple(child for node in frontier for child in _subschemas(node)) + + +def schema_fragment_ref(ref: str) -> str: + return f"#{ref}" if ref.startswith(("/$defs/", "/definitions/")) else ref + + +def schema_has_unresolvable_ref(schema: Mapping[str, object]) -> bool: + definitions: Final = MappingProxyType(dict(_root_definitions(schema))) + if not _refs_in(schema) <= definitions.keys(): + return True + try: + tuple(TopologicalSorter({pointer: _refs_in(body) for pointer, body in definitions.items()}).static_order()) + except CycleError: + return True + return False + + +def schema_union_property_count(schema: Mapping[str, object]) -> int: + return sum(1 for _, node in _property_schemas(schema) if _is_union(node)) + + +def schema_optional_property_count(schema: Mapping[str, object]) -> int: + return sum(len(_optional_property_names(node)) for node in _schema_nodes(schema)) + + +def schema_property_count(schema: Mapping[str, object]) -> int: + return sum(len(properties) for node in _schema_nodes(schema) if (properties := _schema_at(node, "properties"))) + + +def schema_grammar_branch_count(schema: Mapping[str, object]) -> int: + return sum(_grammar_branches(node) for node in _schema_nodes(schema)) + + +def schema_has_open_object(schema: Mapping[str, object]) -> bool: + return any(_is_open_object(node) for node in _schema_nodes(schema)) + + +def schema_has_lookaround_pattern(schema: Mapping[str, object]) -> bool: + return any( + isinstance(pattern := node.get("pattern"), str) and _uses_regex_lookaround(pattern) + for node in _schema_nodes(schema) + ) + + +def schema_string_formats(schema: Mapping[str, object]) -> frozenset[str]: + return frozenset( + value for node in _schema_nodes(schema) if _may_be_string(node) and isinstance(value := node.get("format"), str) + ) + + +def _may_be_string(node: Mapping[str, object]) -> bool: + node_type: Final = node.get("type") + return node_type is None or node_type == "string" or "string" in _list_at(node, "type") + + +def is_object_schema(node: Mapping[str, object]) -> bool: + node_type: Final = node.get("type") + return node_type == "object" or (isinstance(node_type, list) and "object" in node_type) + + +def _schema_nodes(schema: Mapping[str, object]) -> Iterator[Mapping[str, object]]: + return chain.from_iterable(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING)) + + +_SCHEMA_LIST: Final = TypeAdapter(tuple[object, ...]) + + +def _list_at(container: Mapping[str, object], key: str) -> tuple[object, ...]: + value: Final = container.get(key) + return _SCHEMA_LIST.validate_python(value) if isinstance(value, list) else () + + +def _named_schemas(container: Mapping[str, object] | None) -> Iterator[tuple[str, Mapping[str, object]]]: + if container is None: + return iter(()) + return ((key, value) for key in container if (value := _schema_at(container, key)) is not None) + + +def _root_definitions(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]: + for defs_key in ("$defs", "definitions"): + yield from ((f"#/{defs_key}/{name}", body) for name, body in _named_schemas(_schema_at(schema, defs_key))) + + +def _refs_in(schema: Mapping[str, object]) -> frozenset[str]: + return frozenset( + schema_fragment_ref(ref) for node in _schema_nodes(schema) if isinstance(ref := node.get("$ref"), str) + ) + + +def _property_schemas(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]: + return chain.from_iterable(_named_schemas(_schema_at(node, "properties")) for node in _schema_nodes(schema)) + + +def _is_union(node: Mapping[str, object]) -> bool: + return isinstance(node.get("anyOf"), list) or isinstance(node.get("oneOf"), list) or len(_list_at(node, "type")) > 1 + + +def _grammar_branches(node: Mapping[str, object]) -> int: + properties: Final = _schema_at(node, "properties") + if properties is None: + return 0 + optional: Final = len(_optional_property_names(node)) + return (1 << optional) * (len(properties) - optional + 1) + + +def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]: + properties: Final = _schema_at(node, "properties") + if properties is None: + return frozenset() + return frozenset(properties) - frozenset(name for name in _list_at(node, "required") if isinstance(name, str)) + + +def _is_open_object(node: Mapping[str, object]) -> bool: + additional_properties: Final = node.get("additionalProperties") + if additional_properties is True or isinstance(additional_properties, dict) or "patternProperties" in node: + return True + return is_object_schema(node) and "properties" not in node and additional_properties is not False + + +def _subschemas(node: Mapping[str, object]) -> Iterator[Mapping[str, object]]: + for key, value in node.items(): + if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict): + yield from (sub for sub in value.values() if isinstance(sub, dict)) + elif key in _SUBSCHEMA_LIST_KEYWORDS and isinstance(value, list): + yield from (sub for sub in value if isinstance(sub, dict)) + elif key in _SUBSCHEMA_KEYWORDS and isinstance(value, dict): + yield value + + +def _node_without_rejected_regex( + node: Mapping[str, object], + rebuilt: Mapping[int, Mapping[str, object]], + rejected: Callable[[str], bool], +) -> Mapping[str, object]: + kept: Final = { + key: _keyword_value_rebuilt(key, value, rebuilt, rejected) + for key, value in node.items() + if key != "pattern" or not isinstance(value, str) or not rejected(value) + } + if len(kept) == len(node) and all(kept[key] is node[key] for key in kept): + return node + dropped_pattern_properties: Final = _dropped_pattern_properties(node, kept, rebuilt) + if not dropped_pattern_properties or kept.get("additionalProperties") is not False: + return kept + return {**kept, "additionalProperties": _any_of(dropped_pattern_properties)} + + +def _dropped_pattern_properties( + node: Mapping[str, object], + kept: Mapping[str, object], + rebuilt: Mapping[int, Mapping[str, object]], +) -> tuple[object, ...]: + before: Final = _schema_at(node, "patternProperties") + after: Final = _schema_at(kept, "patternProperties") + if before is None or after is None: + return () + return tuple(rebuilt.get(id(sub), sub) for name, sub in before.items() if name not in after) + + +def _schema_at(container: Mapping[str, object], key: str) -> Mapping[str, object] | None: + value: Final = container.get(key) + return value if isinstance(value, dict) else None + + +def _any_of(schemas: tuple[object, ...]) -> object: + return schemas[0] if len(schemas) == 1 else {"anyOf": list(schemas)} + + +def _keyword_value_rebuilt( + key: str, + value: object, + rebuilt: Mapping[int, Mapping[str, object]], + rejected: Callable[[str], bool], +) -> object: + if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict): + kept: Final = { + name: rebuilt.get(id(sub), sub) + for name, sub in value.items() + if key != "patternProperties" or not isinstance(name, str) or not rejected(name) + } + return value if len(kept) == len(value) and all(kept[name] is value[name] for name in kept) else kept + if key in _SUBSCHEMA_LIST_KEYWORDS and isinstance(value, list): + items: Final = [rebuilt.get(id(sub), sub) for sub in value] + return value if all(new is old for new, old in zip(items, value, strict=True)) else items + if key in _SUBSCHEMA_KEYWORDS and isinstance(value, dict): + return rebuilt.get(id(value), value) + return value + + +def _is_not_python_regex(pattern: str) -> bool: + try: + re.compile(pattern) + except (re.error, RecursionError): + return True + return False + + +_REGEX_LOOKAROUND_RE: Final = re.compile(r"\(\? bool: + return _REGEX_LOOKAROUND_RE.search(pattern) is not None + + +def flatten_combinators_and_drop_non_python_regex_patterns(schema: Mapping[str, object]) -> Mapping[str, object]: + return flatten_top_level_schema_combinators(drop_non_python_regex_patterns(schema)) + + +def tool_with_sanitized_parameters( + tool: Mapping[str, object], + sanitize: Callable[[Mapping[str, object]], Mapping[str, object]], +) -> Mapping[str, object]: + """Run the tool's JSON schema through ``sanitize``: ``function.parameters`` on an + OpenAI tool, ``input_schema`` on an Anthropic one. The same object comes back when + nothing changed.""" + function: Final = _schema_at(tool, "function") + if function is not None: + parameters: Final = _schema_at(function, "parameters") + if parameters is None: + return tool + sanitized_parameters: Final = sanitize(parameters) + if sanitized_parameters is parameters: + return tool + return {**tool, "function": {**function, "parameters": sanitized_parameters}} + input_schema: Final = _schema_at(tool, "input_schema") + if input_schema is None: + return tool + sanitized_schema: Final = sanitize(input_schema) + return tool if sanitized_schema is input_schema else {**tool, "input_schema": sanitized_schema} diff --git a/litellm/litellm_core_utils/prompt_templates/prompt_cache_breakpoint_utils.py b/litellm/litellm_core_utils/prompt_templates/prompt_cache_breakpoint_utils.py new file mode 100644 index 00000000000..27cf83210b7 --- /dev/null +++ b/litellm/litellm_core_utils/prompt_templates/prompt_cache_breakpoint_utils.py @@ -0,0 +1,212 @@ +from collections.abc import Mapping +from typing import Final, TypeVar, cast # noqa: TID251 # Existing casts retained unchanged in this pure move + +from pydantic import TypeAdapter, ValidationError + +from litellm import verbose_logger +from litellm.exceptions import UnsupportedParamsError +from litellm.types.llms.openai import ( + ChatCompletionCachedContent, + ChatCompletionPromptCacheBreakpoint, + PromptCacheBreakpoint, +) + +_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object]) + + +PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT: Final = ( + "set `drop_params: true` on the request or in litellm_settings (`litellm.drop_params = True` in the SDK)" +) + + +def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: + """A breakpoint the client already wrote on the block wins, TTL included.""" + if marker is None or target.get("prompt_cache_breakpoint") is not None: + return target + marked: Final = {**target, "prompt_cache_breakpoint": marker} + return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key + + +_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) + + +_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(ChatCompletionPromptCacheBreakpoint) + + +_AcceptedBreakpointT: Final = TypeVar("_AcceptedBreakpointT", bound=Mapping[str, object]) + + +def _accepted_prompt_cache_breakpoint( + accepted: TypeAdapter[_AcceptedBreakpointT], marker: object +) -> _AcceptedBreakpointT | None: + try: + return accepted.validate_python(marker) + except ValidationError: + return None + + +def prompt_cache_breakpoint_within( + accepted: TypeAdapter[_AcceptedBreakpointT], marker: object +) -> _AcceptedBreakpointT | None: + """The client's marker cut down to the ``accepted`` shape: an unsupported ttl is dropped and the mode kept, + and a marker whose mode the target cannot read is dropped whole.""" + whole: Final = _accepted_prompt_cache_breakpoint(accepted, marker) + if whole is not None: + return whole + fields: Final = _breakpoint_fields_or_none(marker) + without_ttl: Final = ( + None + if fields is None + else _accepted_prompt_cache_breakpoint(accepted, {key: item for key, item in fields.items() if key != "ttl"}) + ) + verbose_logger.debug( + "Dropping malformed prompt_cache_breakpoint %r under drop_params" + if without_ttl is None + else "Dropping the unsupported ttl of prompt_cache_breakpoint %r under drop_params", + marker, + ) + return without_ttl + + +def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: + if marker is None or not drop_params: + return marker + return prompt_cache_breakpoint_within(_PROMPT_CACHE_BREAKPOINT, marker) + + +def prompt_cache_breakpoint_with_readable_mode(marker: object, drop_params: bool) -> object: + """The client's marker as written, or None under drop_params when no target could read its mode, so the + target's own config decides which ttl and keys it keeps.""" + return None if prompt_cache_breakpoint_for_wire(marker, drop_params) is None else marker + + +def whole_prompt_cache_breakpoint_or_none(marker: object) -> PromptCacheBreakpoint | None: + return _accepted_prompt_cache_breakpoint(_PROMPT_CACHE_BREAKPOINT, marker) + + +def chat_completion_prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: + if marker is None or not drop_params: + return marker + return prompt_cache_breakpoint_within(_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT, marker) + + +def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: + """Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block.""" + if block.get("cache_control") is None: + return block + unmarked: Final = {key: value for key, value in block.items() if key != "cache_control"} + return cast( # cast-ok: same block shape as the input with the marker key renamed + _MarkedT, with_prompt_cache_breakpoint(unmarked, PromptCacheBreakpoint(mode="explicit")) + ) + + +def prompt_cache_breakpoint_as_cache_control(block: _MarkedT, drop_params: bool) -> _MarkedT: + """OpenAI's ``prompt_cache_breakpoint`` mark rewritten as Anthropic's ``cache_control`` on the same block. + A ``cache_control`` the client already wrote on the block wins, TTL included.""" + marker: Final = block.get("prompt_cache_breakpoint") + if marker is None: + return block + unmarked: Final = {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"} + control: Final = ( + None + if block.get("cache_control") is not None + else _cache_control_for_prompt_cache_breakpoint(marker, drop_params) + ) + marked: Final = unmarked if control is None else {**unmarked, "cache_control": control} + return cast(_MarkedT, marked) # cast-ok: same block shape as the input with the marker key renamed + + +def _cache_control_for_prompt_cache_breakpoint(marker: object, drop_params: bool) -> ChatCompletionCachedContent | None: + try: + breakpoint_mark: Final = _PROMPT_CACHE_BREAKPOINT.validate_python(marker) + except ValidationError: + if not drop_params: + raise UnsupportedParamsError( + message=( + f"prompt_cache_breakpoint must be {{'mode': 'explicit'}} with an optional ttl of '30m' or '12h', got " + f"{marker!r}. Fix the marker, or " + f"{PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it." + ), + status_code=400, + ) + verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker) + return None + ttl: Final = breakpoint_mark.get("ttl") + if ttl is not None and not drop_params: + raise UnsupportedParamsError( + message=( + f"This model's prompt cache lasts 5m, or 1h where it offers one, so prompt_cache_breakpoint " + f"ttl={ttl!r} has no equivalent. Remove the ttl, {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to send " + "a 5m breakpoint instead, or send cache_control {'type': 'ephemeral', 'ttl': '1h'} on the block " + "where the model offers a 1h cache." + ), + status_code=400, + ) + return ChatCompletionCachedContent(type="ephemeral") + + +_BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"}) + + +_BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +_BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object]) + + +def _breakpoint_fields_or_none(value: object) -> Mapping[str, object] | None: + try: + return _BREAKPOINT_FIELDS.validate_python(value) + except ValidationError: + return None + + +def _without_prompt_cache_breakpoint(value: object) -> object: + if not isinstance(value, Mapping): + return value + fields: Final = _BREAKPOINT_FIELDS.validate_python(value) + return {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"} + + +def _breakpoints_stripped_from(key: str, value: object) -> object: + if key in _BREAKPOINT_BLOCK_LIST_KEYS and isinstance(value, list): + return [_without_prompt_cache_breakpoint(block) for block in _BREAKPOINT_BLOCKS.validate_python(value)] + return _without_prompt_cache_breakpoint(value) if key == "function" else value + + +def without_prompt_cache_breakpoints(block: _MarkedT) -> _MarkedT: + """A message or tool definition with every ``prompt_cache_breakpoint`` removed: its own, its content parts', + its tool calls' and its ``function``'s.""" + stripped: Final = { + key: _breakpoints_stripped_from(key, value) for key, value in block.items() if key != "prompt_cache_breakpoint" + } + return cast(_MarkedT, stripped) # cast-ok: same block shape as the input minus the marker keys + + +def _part_with_chat_completion_prompt_cache_breakpoint(part: object) -> object: + if not isinstance(part, Mapping): + return part + fields: Final = _BREAKPOINT_FIELDS.validate_python(part) + marker: Final = fields.get("prompt_cache_breakpoint") + if marker is None: + return fields + unmarked: Final = {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"} + return with_prompt_cache_breakpoint(unmarked, chat_completion_prompt_cache_breakpoint_for_wire(marker, True)) + + +def chat_completion_prompt_cache_breakpoints_for_wire(message: _MarkedT, drop_params: bool) -> _MarkedT: + content: Final = message.get("content") + if not drop_params or not isinstance(content, list): + return message + parts: Final = [ + _part_with_chat_completion_prompt_cache_breakpoint(part) for part in _BREAKPOINT_BLOCKS.validate_python(content) + ] + return cast(_MarkedT, {**message, "content": parts}) # cast-ok: same message shape with its parts' markers trimmed + + +LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS: Final = frozenset( + {"thinking_blocks", "reasoning_items", "provider_specific_fields"} +) + + +LITELLM_INTERNAL_MESSAGE_FIELDS: Final = LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS | {"reasoning_content"} diff --git a/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py b/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py new file mode 100644 index 00000000000..817b1fa5915 --- /dev/null +++ b/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py @@ -0,0 +1,274 @@ +import base64 +import binascii +import re +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from itertools import groupby +from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move + +from litellm.types.llms.openai import ( + ChatCompletionReasoningItem, + ChatCompletionReasoningSummaryTextBlock, + ChatCompletionRedactedThinkingBlock, + ChatCompletionThinkingBlock, +) + + +def _provider_text_or_none(value: object) -> str | None: + return cast(str | None, value) # cast-ok: reasoning fields arrive in untyped provider messages + + +def extract_reasoning_content(message: Mapping[str, object]) -> tuple[str | None, str | None]: + """ + Extract reasoning content and main content from a message. + + Args: + message (dict): The message dictionary that may contain reasoning_content + + Returns: + tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content) + """ + message_content: Final = message.get("content") + if "reasoning_content" in message: + return _provider_text_or_none(message["reasoning_content"]), _provider_text_or_none(message_content) + elif "reasoning" in message: + return _provider_text_or_none(message["reasoning"]), _provider_text_or_none(message_content) + elif isinstance(message_content, str): + return parse_content_for_reasoning(message_content) + return None, _provider_text_or_none(message_content) + + +_extract_reasoning_content = extract_reasoning_content + + +def _readable_thinking_text(block: Mapping[str, object]) -> str: + """The text a chat model can read back, empty for redacted blocks and malformed ones.""" + if block.get("type") != "thinking": + return "" + return str(block.get("thinking") or "") + + +def reasoning_content_from_thinking_blocks( + thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock], +) -> str: + """Flatten Anthropic thinking blocks into the `reasoning_content` string chat models expect. + + Redacted blocks carry no readable text, so they contribute nothing. + """ + return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block))) + + +ENCRYPTED_REASONING_SIGNATURE_PREFIX: Final = "litellm_encrypted_reasoning:" + + +def encrypted_reasoning_signature(encrypted_content: str) -> str: + """The opaque value a Responses API reasoning item's `encrypted_content` travels in. + + Anthropic clients echo a thinking block's `signature` and a redacted block's `data` + back verbatim, so either field can carry the encrypted reasoning across turns; the + prefix tells the two apart from a signature Anthropic minted. + """ + return f"{ENCRYPTED_REASONING_SIGNATURE_PREFIX}{encrypted_content}" + + +def _carries_encrypted_reasoning(signature: object) -> bool: + return isinstance(signature, str) and signature.startswith(ENCRYPTED_REASONING_SIGNATURE_PREFIX) + + +def encrypted_content_from_signature(signature: object) -> str | None: + if not isinstance(signature, str) or not _carries_encrypted_reasoning(signature): + return None + return signature.removeprefix(ENCRYPTED_REASONING_SIGNATURE_PREFIX) or None + + +def _encrypted_reasoning_field(block: Mapping[str, object]) -> object: + match block.get("type"): + case "thinking": + return block.get("signature") + case "redacted_thinking": + return block.get("data") + case _: + return None + + +def encrypted_content_of_block(block: Mapping[str, object]) -> str | None: + return encrypted_content_from_signature(_encrypted_reasoning_field(block)) + + +def is_encrypted_reasoning_block(block: object) -> bool: + """A thinking or redacted_thinking block carrying Responses API encrypted reasoning. + + Only the Responses API that minted the content can read it back, so an Anthropic + backend has to drop such a block rather than fail signature verification on it. + """ + if not isinstance(block, Mapping): + return False + mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance + return _carries_encrypted_reasoning(_encrypted_reasoning_field(mapping)) + + +def is_unsignable_thinking_block(block: object) -> bool: + """A thinking block Anthropic cannot accept on input. + + Anthropic verifies the signature cryptographically, so a block with a null, + empty, or missing signature (e.g. from an open-source reasoning model) is + rejected with a 400, and so is a block whose signature or data carries + another provider's encrypted reasoning. A signed block is kept whatever its + text, since ``display: omitted`` returns signed blocks with empty ``thinking``. + `redacted_thinking` blocks carry no signature and are always kept. + """ + if is_encrypted_reasoning_block(block): + return True + if not isinstance(block, Mapping): + return False + mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance + if mapping.get("type") != "thinking": + return False + signature: Final = mapping.get("signature") + return not (isinstance(signature, str) and len(signature) > 0) + + +def has_blank_redacted_data(block: object) -> bool: + match block: + case {"type": "redacted_thinking", "data": str() as data}: + return not data.strip() + case {"type": "redacted_thinking"}: + return True + case _: + return False + + +def native_redacted_thinking_data(data: str) -> str: + """Claude's own `data` string for a redacted_thinking block. + + Bedrock Converse carries it base64-wrapped as `redactedContent`, and LiteLLM passed that + wrapper through as `data` until it started decoding Converse replies, so histories saved + before then still hold it. Claude's own value never decodes twice, since its first byte + decodes to a protobuf tag outside the base64 alphabet. + """ + return _converse_wrapped_payload(data) or data + + +def converse_redacted_content(data: str) -> str: + """The `redactedContent` Bedrock Converse expects for a redacted_thinking block's `data`.""" + return base64.b64encode(native_redacted_thinking_data(data).encode()).decode() + + +def _converse_wrapped_payload(data: str) -> str | None: + try: + payload: Final = base64.b64decode(data, validate=True).decode("ascii") + _ = base64.b64decode(payload, validate=True) + except (binascii.Error, ValueError): + return None + else: + return payload + + +def strip_encrypted_reasoning_from_messages( + messages: object, + *, + should_strip: Callable[[Mapping[str, object]], bool] | None = None, +) -> None: + """Drop the bridge-tagged reasoning blocks a routed deployment cannot decrypt from + Anthropic-shaped history. + + The whole block goes, the way #40280 drops undecryptable Responses ``input`` items: a + provider that did not mint the block rejects it signed (a foreign signature) and unsigned + (a missing signature) alike, so keeping its text as an unsigned thinking block only moves + the 400 from the router to the provider. + + Mutates the content lists in place: the router's fallback snapshot shares these + message objects, so a rebound list would replay the stripped blocks on the fallback hop. + """ + if not isinstance(messages, list): + return + for content in anthropic_content_lists(cast(list[object], messages)): # cast-ok: untyped client json + _strip_encrypted_reasoning_from_blocks(content, should_strip=should_strip) + + +def anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]: + return ( + cast(list[object], content) # cast-ok: narrowed by isinstance + for message in messages + if isinstance(message, Mapping) + for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance + if isinstance(content, list) + ) + + +def _strip_encrypted_reasoning_from_blocks( + content: object, + *, + should_strip: Callable[[Mapping[str, object]], bool] | None = None, +) -> None: + blocks: Final = cast(list[object], content) # cast-ok: narrowed by the caller's isinstance + kept: Final = tuple( + block + for block in blocks + if not is_encrypted_reasoning_block(block) + or (should_strip is not None and not should_strip(cast(Mapping[str, object], block))) + ) + blocks[:] = kept + + +def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: + index, block = indexed_block + return f"encrypted:{index}" if is_encrypted_reasoning_block(block) else "summary" + + +def _reasoning_item_from_block_group(group: tuple[Mapping[str, object], ...]) -> ChatCompletionReasoningItem | None: + summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [ # mutable-ok: API message payload + ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text) + for block in group + if (text := _readable_thinking_text(block)) + ] + encrypted_content: Final = encrypted_content_of_block(group[0]) + if encrypted_content is not None: + return ChatCompletionReasoningItem(type="reasoning", summary=summary, encrypted_content=encrypted_content) + if not summary: + return None + return ChatCompletionReasoningItem(type="reasoning", summary=summary) + + +def responses_reasoning_items_from_thinking_blocks( + thinking_blocks: Iterable[Mapping[str, object]], +) -> tuple[ChatCompletionReasoningItem, ...]: + """Build Responses API `reasoning` input items from Anthropic thinking blocks. + + A block carrying encrypted reasoning replays the item it came from byte for byte; + a run of plain thinking blocks collapses into one summary-only item. No item carries + an `id`: the Responses API 404s on any id it did not mint itself and rejects an empty + one, while an item without an id is always accepted. + """ + return tuple( + item + for _, group in groupby(enumerate(thinking_blocks), key=_reasoning_replay_group_key) + if (item := _reasoning_item_from_block_group(tuple(block for _, block in group))) is not None + ) + + +def parse_content_for_reasoning( + message_text: str | None, +) -> tuple[str | None, str | None]: + """ + Parse the content for reasoning + + Returns: + - reasoning_content: The content of the reasoning + - content: The content of the message + """ + if not message_text: + return None, message_text + + reasoning_match: Final = re.match( + r"<(?:think|thinking|budget:thinking)>(.*?)(.*)", + message_text, + re.DOTALL, + ) + + if reasoning_match: + return reasoning_match.group(1), reasoning_match.group(2) + + return None, message_text + + +_parse_content_for_reasoning = parse_content_for_reasoning diff --git a/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py b/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py new file mode 100644 index 00000000000..317ffa7b3b1 --- /dev/null +++ b/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py @@ -0,0 +1,252 @@ +import json +from typing import Any, Final # noqa: TID251 # Existing annotations retained unchanged in this pure move + +from pydantic import TypeAdapter + +from litellm import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping + +_DECODED_JSON: Final = TypeAdapter(object) + + +def _attempt_json_repair(s: str) -> object | None: + """ + Attempt to repair truncated JSON produced by LLM tool calls. + + Handles the most common truncation patterns where the model generates + valid JSON that is cut short (missing closing brackets/braces). + + Returns the parsed value on success, or None if repair fails. + """ + import json + + stripped: Final = s.rstrip() + if not stripped: + return None + + # Track the stack of unmatched openers to respect nesting order + opener_stack: Final[list] = [] + in_string = False + escape_next = False + + for ch in stripped: + if escape_next: + escape_next = False + continue + if ch == "\\": + if in_string: + escape_next = True + continue + if ch == '"': + in_string = not in_string + continue + if in_string: + continue + if ch == "{": + opener_stack.append("}") + elif ch == "[": + opener_stack.append("]") + elif ch in ("}", "]"): + if opener_stack and opener_stack[-1] == ch: + opener_stack.pop() + + if not opener_stack: + return None + + # Remove trailing comma before we close brackets + candidate = stripped.rstrip(",") + + # Close in reverse order of opening (respects nesting) + candidate += "".join(reversed(opener_stack)) + + try: + return _DECODED_JSON.validate_python(json.loads(candidate)) + except json.JSONDecodeError: + pass + + return None + + +def parse_tool_call_arguments( + arguments: str | None, + tool_name: str | None = None, + context: str | None = None, +) -> Any: + """ + Parse tool call arguments from a JSON string. + + When the JSON is malformed (e.g. truncated by the model), this function + attempts a lightweight repair (closing unmatched brackets/braces) before + raising an error. A warning is logged whenever repair succeeds so that + callers are aware the arguments were not perfectly formed. + + Args: + arguments: The JSON string containing tool arguments, or None. + tool_name: Optional name of the tool (for error messages). + context: Optional context string (e.g., "Anthropic Messages API"). + + Returns: + Parsed arguments (usually a dict, but may be any JSON-deserializable + type such as list, str, int, float, or None). Returns empty dict if + arguments is None or empty. + + Raises: + ValueError: If the arguments string is not valid JSON and cannot be repaired. + """ + import json + + if not arguments or not arguments.strip(): + return {} + + try: + return _DECODED_JSON.validate_python(json.loads(arguments)) + except json.JSONDecodeError as original_error: + repaired: Final = _attempt_json_repair(arguments) + if repaired is not None: + verbose_logger.warning( + "Repaired truncated tool call arguments for tool '%s' (%s). Original (%d chars): %.200s%s", + tool_name or "", + context or "unknown context", + len(arguments), + arguments, + "..." if len(arguments) > 200 else "", + ) + return repaired + + error_parts: Final = ["Failed to parse tool call arguments"] + + if tool_name: + error_parts.append(f"for tool '{tool_name}'") + if context: + error_parts.append(f"({context})") + + error_message: Final = " ".join(error_parts) + f". Error: {original_error}. Arguments: {arguments}" + + raise ValueError(error_message) from original_error + + +def split_concatenated_json_objects(raw: str) -> list[dict[str, object]]: + """ + Split a string that contains one or more concatenated JSON objects into + a list of parsed dicts. + + LLM providers (notably Bedrock Claude Sonnet 4.5) sometimes return + multiple tool-call argument objects concatenated in a single + ``arguments`` string, e.g.:: + + '{"command":["curl",...]}{"command":["curl",...]}{"command":["curl",...]}' + + ``json.loads()`` fails on this with ``JSONDecodeError: Extra data``. + This helper uses ``json.JSONDecoder.raw_decode()`` to walk the string + and extract each JSON object individually. + + The walk degrades gracefully: if the string is malformed or truncated + (e.g. a stream that ended mid-tool-call), whatever complete objects were + parsed before the bad tail are returned and the remainder is discarded + with a warning, rather than raising. The sole caller + (``_convert_to_bedrock_tool_call_invoke``) treats an empty result as + ``input={}`` so the conversation can continue instead of hard-failing. + + Returns + ------- + list[dict] + A list of parsed dicts – one per JSON object found. If *raw* is + empty, whitespace-only, or wholly unparseable, an empty list is + returned. + """ + import json + + raw = raw.strip() + if not raw: + return [] + + decoder: Final = json.JSONDecoder() + results: Final[list[dict[str, object]]] = [] + idx = 0 + length: Final = len(raw) + + while idx < length: + # Skip whitespace between objects + while idx < length and raw[idx] in " \t\n\r": + idx += 1 + if idx >= length: + break + + try: + decoded = decoder.raw_decode(raw, idx) + except json.JSONDecodeError as e: + verbose_logger.warning( + "split_concatenated_json_objects: discarding unparseable tool-call " + "arguments tail after %d complete object(s); decode_start=%d error=%s", + len(results), + idx, + e, + ) + break + obj = as_str_mapping(_DECODED_JSON.validate_python(decoded[0])) + if obj is not None: + results.append(dict(obj)) + else: + # Non-dict JSON value – wrap in empty dict (Bedrock requires + # toolUse.input to be an object). + results.append({}) + idx = decoded[1] + + return results + + +MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS: Final = 8 + + +def salvage_concatenated_tool_arguments(raw: str) -> tuple[dict[str, object], ...]: + """Return complete concatenated JSON objects that are safe to expand. + + Identical objects collapse to the first one and are not capped. More than + ``MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS`` objects that are not all identical + returns an empty tuple. Anything that is not a full concatenation of JSON + objects returns an empty tuple. Repeated copies of the first object are not + retained, and once the cap is passed the rest of the string is only checked. + """ + stripped: Final = raw.strip() + if not stripped: + return () + decoder: Final = json.JSONDecoder() + length: Final = len(stripped) + idx = 0 # rebind-ok: cursor walks the concatenated JSON string + count = 0 # rebind-ok: counts complete objects without retaining duplicates + kept = () # rebind-ok: holds at most one object past the salvage cap + exceeded = False # rebind-ok: cap already passed, the tail is only validated + while idx < length: + while idx < length and stripped[idx] in " \t\n\r": + idx += 1 + if idx >= length: + break + try: + decoded = decoder.raw_decode(stripped, idx) + except json.JSONDecodeError: + return () + decoded_object = as_str_mapping(_DECODED_JSON.validate_python(decoded[0])) + if decoded_object is None: + return () + idx = decoded[1] + if exceeded: + continue + obj = dict(decoded_object) + count += 1 + if not kept: + kept = (obj,) + continue + if obj == kept[0] and len(kept) == 1: + continue + if len(kept) == 1 and count > 2 and count - 1 > MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS: + exceeded = True + continue + if len(kept) == 1 and count > 2: + kept = (kept[0],) * (count - 1) + if len(kept) >= MAX_SALVAGED_TOOL_ARGUMENT_OBJECTS: + exceeded = True + continue + kept = (*kept, obj) + if exceeded: + return () + return kept diff --git a/litellm/litellm_core_utils/prompt_templates/tpm_media_utils.py b/litellm/litellm_core_utils/prompt_templates/tpm_media_utils.py new file mode 100644 index 00000000000..a4cb52d689c --- /dev/null +++ b/litellm/litellm_core_utils/prompt_templates/tpm_media_utils.py @@ -0,0 +1,259 @@ +import base64 +import re +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from itertools import chain +from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move + +from litellm.litellm_core_utils.core_helpers import as_str_mapping + + +@dataclass(frozen=True, slots=True) +class _TPMMediaProjection: + content: object + media: tuple[Mapping[str, object], ...] = () + + +def _tpm_image_part(url: str, detail: object = "auto") -> Mapping[str, object]: + return {"type": "image_url", "image_url": {"url": url, "detail": detail}} + + +def _tpm_blank_fields(fields: Mapping[str, object], names: tuple[str, ...]) -> Mapping[str, object]: + return {key: "" if key in names and isinstance(value, str) else value for key, value in fields.items()} + + +def _tpm_file_media(part: Mapping[str, object]) -> _TPMMediaProjection: + from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type + + nested: Final = part.get("type") == "file" + fields: Final = as_str_mapping(part.get("file")) if nested else part + if fields is None: + return _TPMMediaProjection(part) + raw_data: Final = fields.get("file_data") + data: Final = raw_data if isinstance(raw_data, str) else "" + declared: Final = next( + ( + value + for key in ("format", "mime_type", "content_type") + if isinstance(value := fields.get(key), str) and value + ), + "", + ) + mime: Final = (declared or parse_mime_type(data[:256]) or "").split(";", 1)[0].strip().lower() + if not mime.startswith(("image/", "audio/")): + return _TPMMediaProjection(part) + raw_url: Final = fields.get("file_url") or fields.get("file_id") + url: Final = data or (raw_url if isinstance(raw_url, str) else "") + header_end: Final = data.find(",", 0, 256) + encoded: Final = ( + data[header_end + 1 :] + if header_end >= 0 and data[:header_end].endswith(";base64") + else "" + if data.startswith(("data:", "https://", "http://", "gs://")) + else data + ) + media: Final = ( + _tpm_image_part( + url + if not data or url.startswith(("data:", "https://", "http://", "gs://")) + else f"data:{mime};base64,{url}", + fields.get("detail", "auto"), + ) + if mime.startswith("image/") + else {"type": "input_audio", "input_audio": {"data": encoded, "format": mime.removeprefix("audio/")}} + ) + projected: Final = _tpm_blank_fields(fields, ("file_data", "file_url")) + return _TPMMediaProjection({**part, "file": projected} if nested else projected, (media,)) + + +def _tpm_media_leaf(value: object) -> _TPMMediaProjection: + part: Final = as_str_mapping(value) + if part is None: + return _TPMMediaProjection(value) + kind: Final = part.get("type") + if kind in ("file", "input_file"): + return _tpm_file_media(part) + if kind == "input_audio": + audio: Final = as_str_mapping(part.get("input_audio")) + return _TPMMediaProjection( + {**part, "input_audio": _tpm_blank_fields(audio, ("data",))} if audio is not None else value, + ({"type": "input_audio", "input_audio": part.get("input_audio")},), + ) + if kind in ("image_url", "input_image", "image", "computer_screenshot"): + image: Final = as_str_mapping(part.get("image_url")) + source: Final = as_str_mapping(part.get("source")) + canonical: Final = _tpm_image_part( + _image_part_url({**part, "type": "input_image"} if kind == "computer_screenshot" else part) or "", + (image or part).get("detail", part.get("detail", "auto")), + ) + projected: Final = ( + {**part, "image_url": _tpm_blank_fields(image, ("url",))} + if image is not None and kind in ("image_url", "input_image", "computer_screenshot") + else {**part, "source": _tpm_blank_fields(source, ("data", "url"))} + if source is not None and kind == "image" and source.get("type") in ("base64", "url") + else _tpm_blank_fields(part, ("image_url", "url")) + ) + return _TPMMediaProjection(projected, (canonical,)) + return _TPMMediaProjection(value) + + +def _tpm_media_list(value: object, split: Callable[[object], _TPMMediaProjection]) -> _TPMMediaProjection: + if not isinstance(value, (list, tuple)): + return _TPMMediaProjection(value) + parts: Final = cast(Sequence[object], value) # cast-ok: finite JSON content arrays + projected: Final = tuple(map(split, parts)) + media: Final = tuple(chain.from_iterable(part.media for part in projected)) + content: Final = tuple(part.content for part in projected) + return _TPMMediaProjection(parts if not media else content if isinstance(value, tuple) else list(content), media) + + +def _tpm_replace_field(part: Mapping[str, object], key: str, projection: _TPMMediaProjection) -> _TPMMediaProjection: + return _TPMMediaProjection({**part, key: projection.content} if projection.media else part, projection.media) + + +def _tpm_document_media(value: object) -> _TPMMediaProjection: + part: Final = as_str_mapping(value) + source: Final = as_str_mapping(part.get("source")) if part is not None else None + if part is None or part.get("type") != "document" or source is None or source.get("type") != "content": + return _tpm_media_leaf(value) + content: Final = _tpm_media_list(source.get("content"), _tpm_media_leaf) + return _tpm_replace_field(part, "source", _tpm_replace_field(source, "content", content)) + + +def _tpm_tool_result_media(value: object) -> _TPMMediaProjection: + part: Final = as_str_mapping(value) + if part is None or part.get("type") != "tool_result": + return _tpm_document_media(value) + return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_document_media)) + + +def _tpm_code_output_media(value: object) -> _TPMMediaProjection: + part: Final = as_str_mapping(value) + url: Final = part.get("url") if part is not None else None + if part is None or part.get("type") != "image" or not isinstance(url, str) or not url: + return _TPMMediaProjection(value) + return _TPMMediaProjection(_tpm_blank_fields(part, ("url",)), (_tpm_image_part(url),)) + + +def _tpm_input_media(value: object) -> _TPMMediaProjection: + part: Final = as_str_mapping(value) + if part is None: + return _TPMMediaProjection(value) + kind: Final = part.get("type") + if kind in ("function_call_output", "custom_tool_call_output"): + return _tpm_replace_field(part, "output", _tpm_media_list(part.get("output"), _tpm_tool_result_media)) + if kind == "computer_call_output": + return _tpm_replace_field(part, "output", _tpm_media_leaf(part.get("output"))) + if kind == "code_interpreter_call": + return _tpm_replace_field(part, "outputs", _tpm_media_list(part.get("outputs"), _tpm_code_output_media)) + result: Final = part.get("result") + if kind == "image_generation_call" and isinstance(result, str) and result: + return _TPMMediaProjection( + _tpm_blank_fields(part, ("result",)), + (_tpm_image_part("data:application/octet-stream;base64," + result),), + ) + if kind == "message" or (kind is None and isinstance(part.get("role"), str)): + return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_tool_result_media)) + return _tpm_tool_result_media(value) + + +def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]: + projection: Final = _tpm_media_list(value, _tpm_input_media) + return projection.content, projection.media + + +_TPM_EMBEDDING_FILE_MIME_PREFIXES: Final = ("application/pdf", "video/") + + +def _tpm_embedding_file(data_uri: str) -> tuple[Mapping[str, object], ...]: + from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type + + mime: Final = (parse_mime_type(data_uri[:256]) or "").strip().lower() + if not mime.startswith(_TPM_EMBEDDING_FILE_MIME_PREFIXES): + return () + return ({"type": "file", "file": {"file_data": data_uri, "format": mime}},) + + +_BASE64_PAYLOAD: Final = re.compile(r"[A-Za-z0-9+/]*={0,2}") + + +def _is_base64_data_uri(value: str) -> bool: + header_end: Final = value.find(",", 0, 256) + if header_end < 0 or not value[:header_end].endswith(";base64"): + return False + payload_start: Final = header_end + 1 + if (len(value) - payload_start) % 4 or _BASE64_PAYLOAD.fullmatch(value, payload_start) is None: + return False + last_quantum: Final = value[max(payload_start, len(value) - 4) :] + return base64.b64encode(base64.b64decode(last_quantum)).decode("ascii") == last_quantum + + +def _tpm_embedding_file_block(part: Mapping[str, object]) -> _TPMMediaProjection: + projection: Final = _tpm_file_media(part) + fields: Final = as_str_mapping(part.get("file")) + data: Final = fields.get("file_data") if fields is not None else None + if projection.media or fields is None or not isinstance(data, str) or not _is_base64_data_uri(data): + return projection + media: Final = _tpm_embedding_file(data) + return _TPMMediaProjection({**part, "file": _tpm_blank_fields(fields, ("file_data",))} if media else part, media) + + +def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection: + part: Final = as_str_mapping(value) + if part is not None and part.get("type") == "file": + return _tpm_embedding_file_block(part) + if not isinstance(value, str) or not value.startswith("data:"): + return _tpm_media_leaf(value) + if not _is_base64_data_uri(value): + return _TPMMediaProjection(value) + media: Final = _tpm_file_media({"file_data": value}).media or _tpm_embedding_file(value) + return _TPMMediaProjection("" if media else value, media) + + +def _is_json_array(value: object) -> bool: + return isinstance(value, (list, tuple)) + + +def _tpm_embedding_item(value: object) -> _TPMMediaProjection: + return _tpm_media_list(value, _tpm_embedding_leaf) if _is_json_array(value) else _tpm_embedding_leaf(value) + + +def split_embedding_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]: + projection: Final = ( + _tpm_media_list(value, _tpm_embedding_item) if _is_json_array(value) else _tpm_embedding_leaf(value) + ) + return projection.content, projection.media + + +def embedding_input_is_all_media(value: object) -> bool: + items: Final = cast(Sequence[object], value) if _is_json_array(value) else (value,) # cast-ok: JSON input array + return all(_tpm_embedding_leaf(item).media for item in items) + + +def anthropic_image_source_to_openai_url(image_source: Mapping[str, object]) -> str | None: + """Data or remote URL for an Anthropic ``source`` block, in the form chat completions expects.""" + source_type: Final = image_source.get("type") + if source_type == "base64": + media_type: Final = image_source.get("media_type") or "image/jpeg" + image_data: Final = image_source.get("data") or "" + return f"data:{media_type};base64,{image_data}" if image_data else None + if source_type == "url": + url: Final = image_source.get("url") + return url if isinstance(url, str) else "" + return None + + +def _image_part_url(part: Mapping[str, object]) -> str | None: + """The image URL carried by one content part, whichever of the three dialects wrote it.""" + part_type: Final = part.get("type") + if part_type in ("image_url", "input_image"): + image_url: Final = part.get("image_url") or part.get("url") + if isinstance(image_url, str): + return image_url + image_fields: Final = as_str_mapping(image_url) + url: Final = image_fields.get("url") if image_fields is not None else None + return url if isinstance(url, str) else None + if part_type == "image": + source: Final = part.get("source") + return anthropic_image_source_to_openai_url(source) if isinstance(source, Mapping) else None + return None diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 52a5026c44f..e56dd83a277 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -706,7 +706,7 @@ def _data_uri_media_scope(model: str, media: Sequence[Mapping[str, object]]) -> def split_embedding_input_tokens( value: _EmbeddingInputT, media_models: Callable[[], Sequence[str]], fallback_model: str | None = None ) -> tuple[_EmbeddingInputT, int]: - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( embedding_input_is_all_media, split_embedding_media_for_tpm, ) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index ac4addb0847..a6f6ec3d598 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -37,18 +37,9 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.prompt_templates.common_utils import ( DEFS_MAX_INLINED_BYTES, invalid_type_message, - is_object_schema, is_present_non_string, is_unhashable_value_behind, sanitize_input_schema_for_anthropic, - schema_fragment_ref, - schema_grammar_branch_count, - schema_has_lookaround_pattern, - schema_has_open_object, - schema_has_unresolvable_ref, - schema_optional_property_count, - schema_string_formats, - schema_union_property_count, unpack_defs, ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( @@ -59,6 +50,17 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( inline_remote_image_urls, is_unformatted_pdf_link_image, ) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + is_object_schema, + schema_fragment_ref, + schema_grammar_branch_count, + schema_has_lookaround_pattern, + schema_has_open_object, + schema_has_unresolvable_ref, + schema_optional_property_count, + schema_string_formats, + schema_union_property_count, +) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( message_field, parts_of, diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index bc9bd769d83..275cc5cbe5c 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -29,17 +29,19 @@ from litellm.constants import ( ) from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.core_helpers import as_str_mapping, lax_bool -from litellm.litellm_core_utils.prompt_templates.common_utils import ( - get_file_ids_from_messages, - is_encrypted_reasoning_block, - native_redacted_thinking_data, -) from litellm.litellm_core_utils.prompt_templates.factory import THOUGHT_SIGNATURE_SEPARATOR +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_file_ids_from_messages, +) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( find_anthropic_server_tool_result, message_field, parts_of, ) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + is_encrypted_reasoning_block, + native_redacted_thinking_data, +) from litellm.litellm_core_utils.reasoning_effort_utils import DroppedThinkingBudget, non_numeric_thinking_budget_error from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.wif import ( diff --git a/litellm/llms/anthropic/files/transformation.py b/litellm/llms/anthropic/files/transformation.py index db14d3366ff..98548a58481 100644 --- a/litellm/llms/anthropic/files/transformation.py +++ b/litellm/llms/anthropic/files/transformation.py @@ -21,7 +21,9 @@ import httpx from openai.types.file_deleted import FileDeleted from pydantic import ConfigDict, TypeAdapter -from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, +) from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.transformation import ( diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index afcd94c5b2c..5379dcd6d68 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -123,15 +123,23 @@ def create_tool_name_mapping( from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingChoice from litellm.litellm_core_utils.prompt_templates.common_utils import ( - anthropic_image_source_to_openai_url, anthropic_inline_document, - parse_tool_call_arguments, - reasoning_content_from_thinking_blocks, - with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, ) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + with_prompt_cache_breakpoint, +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + reasoning_content_from_thinking_blocks, +) +from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( + parse_tool_call_arguments, +) +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( + anthropic_image_source_to_openai_url, +) from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, reasoning_effort_from_thinking_silently, diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index ec09987837c..85f042f621c 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -20,7 +20,7 @@ from litellm.litellm_core_utils.asyncify import ( run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( cache_control_as_prompt_cache_breakpoint, prompt_cache_breakpoint_with_readable_mode, with_prompt_cache_breakpoint, diff --git a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py index f0c3316c509..5c899e8e8bf 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py @@ -12,7 +12,7 @@ from litellm import verbose_logger from litellm._logging import redact_internal_details_from_client_message from litellm._uuid import uuid from litellm.exceptions import APIError, MidStreamFallbackError -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, ) from litellm.llms.anthropic.messages.native.utils import ( diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index 0630bef0311..110e9fc40ff 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -14,9 +14,13 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, anthropic_inline_document, +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + with_prompt_cache_breakpoint, +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, responses_reasoning_items_from_thinking_blocks, - with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 0fe7caa1e16..ca036fa4bdf 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -14,17 +14,21 @@ from litellm.integrations.anthropic_cache_control_hook import deployment_prompt_ from litellm.litellm_core_utils.allowed_tools_tool_choice import is_allowed_tools_tool_choice from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( - chat_completion_prompt_cache_breakpoints_for_wire, drop_tool_reference_parts_from_tool_messages, - flatten_combinators_and_drop_non_python_regex_patterns, hoist_images_from_tool_messages, system_messages_first, - tool_with_sanitized_parameters, - without_prompt_cache_breakpoints, ) from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_azure_openai_messages, ) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_combinators_and_drop_non_python_regex_patterns, + tool_with_sanitized_parameters, +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + chat_completion_prompt_cache_breakpoints_for_wire, + without_prompt_cache_breakpoints, +) from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.chat.gpt_reasoning_transformation import GPT_REASONING_SERIES_MARKERS diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index f95dc1e4388..a814a9fbac0 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -11,7 +11,9 @@ from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION -from litellm.litellm_core_utils.prompt_templates.common_utils import prompt_cache_breakpoint_within +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + prompt_cache_breakpoint_within, +) from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config from litellm.llms.azure.common_utils import BaseAzureLLM diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 487958d6d55..93ff6d8170a 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -21,6 +21,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( drop_tool_reference_parts_from_tool_messages, filter_value_from_dict, is_pdf_file_part, +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( prompt_cache_breakpoint_for_wire, with_prompt_cache_breakpoint, ) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 5281e6ae670..b93d40738ea 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -38,25 +38,27 @@ from litellm.litellm_core_utils.core_helpers import ( safe_deep_copy, ) from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_lookaround_regex_patterns, - is_object_schema, - native_redacted_thinking_data, - parse_content_for_reasoning, - tool_with_sanitized_parameters, -) from litellm.litellm_core_utils.prompt_templates.factory import ( BedrockConverseMessagesProcessor, _bedrock_converse_messages_pt, _bedrock_tools_pt, make_valid_bedrock_tool_name, ) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_lookaround_regex_patterns, + is_object_schema, + tool_with_sanitized_parameters, +) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( CONVERTED_SYSTEM_NOTE, is_system_message, message_field, parts_of, ) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + native_redacted_thinking_data, + parse_content_for_reasoning, +) from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.llms.anthropic.chat.transformation import ( diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index a6b34010297..bc2ef2441d2 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -16,7 +16,9 @@ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers from litellm.litellm_core_utils.optional_imports import ensure_optional_import -from litellm.litellm_core_utils.prompt_templates.common_utils import native_redacted_thinking_data +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + native_redacted_thinking_data, +) from litellm.llms.anthropic.chat.handler import ( ModelResponseIterator as AnthropicModelResponseIterator, ) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py index 16dde81ca47..eac9843331c 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Final, cast from httpx import Response from litellm import verbose_logger -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( parse_content_for_reasoning, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 3385c89c3a7..ace7b0735f6 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -35,7 +35,9 @@ from litellm import verbose_logger from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.core_helpers import lax_bool from litellm.litellm_core_utils.optional_imports import ensure_optional_import -from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_property_count, +) from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.base_llm.anthropic_messages.transformation import ( diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index dcdd9e39c20..0fd61eb4c07 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -39,9 +39,11 @@ from litellm.litellm_core_utils.cloud_storage_security import ( validate_managed_cloud_file_id, ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_file_data, text_completion_prompt_to_messages, ) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, +) from litellm.llms.anthropic.common_utils import AnthropicError from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py index 1f7b1712d68..7190a4f28c6 100644 --- a/litellm/llms/bedrock_mantle/responses/transformation.py +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -24,7 +24,9 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import drops_unsupported_params -from litellm.litellm_core_utils.prompt_templates.common_utils import chat_completion_prompt_cache_breakpoint_for_wire +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + chat_completion_prompt_cache_breakpoint_for_wire, +) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items diff --git a/litellm/llms/custom_httpx/container_handler.py b/litellm/llms/custom_httpx/container_handler.py index a9b96217379..e89bceb043a 100644 --- a/litellm/llms/custom_httpx/container_handler.py +++ b/litellm/llms/custom_httpx/container_handler.py @@ -210,7 +210,7 @@ def _prepare_multipart_file_upload( Returns: Tuple of (files_dict, headers_without_content_type) """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( extract_file_data, ) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 859211bd40d..a173cdb50f2 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -19,12 +19,14 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo should_convert_tool_call_to_json_mode, ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_reasoning_content, merge_consecutive_system_messages, strip_litellm_internal_message_fields, strip_name_from_message, strip_prefill_prefix_field, ) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + extract_reasoning_content, +) from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator diff --git a/litellm/llms/gemini/files/transformation.py b/litellm/llms/gemini/files/transformation.py index f9a7e230ea9..17460c0295a 100644 --- a/litellm/llms/gemini/files/transformation.py +++ b/litellm/llms/gemini/files/transformation.py @@ -14,7 +14,9 @@ from openai.types.file_deleted import FileDeleted from typing_extensions import ReadOnly, Required from litellm._logging import verbose_logger -from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, +) from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.files.transformation import ( BaseFilesConfig, diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 3386cd452da..c5a0040eb80 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -7,10 +7,10 @@ import json from collections.abc import Coroutine from typing import Final, Literal, cast, overload -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( get_image_mime_type_from_url, ) -from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, ) diff --git a/litellm/llms/manus/files/transformation.py b/litellm/llms/manus/files/transformation.py index 991aa4cbc32..2db9f827a64 100644 --- a/litellm/llms/manus/files/transformation.py +++ b/litellm/llms/manus/files/transformation.py @@ -20,7 +20,9 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, +) from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.transformation import ( diff --git a/litellm/llms/mistral/files/transformation.py b/litellm/llms/mistral/files/transformation.py index 908567e88fb..1924e80677f 100644 --- a/litellm/llms/mistral/files/transformation.py +++ b/litellm/llms/mistral/files/transformation.py @@ -17,7 +17,9 @@ from openai.types.file_deleted import FileDeleted from pydantic import ConfigDict from typing_extensions import ReadOnly, TypedDict -from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, +) from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.transformation import BaseFilesConfig, LiteLLMLoggingObj diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 59020b058a2..c6f3a7a8280 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -12,6 +12,8 @@ from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, extract_images_from_message, +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( extract_reasoning_content, ) from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE @@ -353,7 +355,7 @@ class OllamaChatConfig(BaseConfig): del response_json_message["thinking"] elif response_json_message.get("content") is not None: # parse reasoning content from content - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( parse_content_for_reasoning, ) diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 6dbcb050e11..565444f3f48 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -75,7 +75,7 @@ class _OllamaGenerateReasoning(LiteLLMBaseModel): def split(self) -> tuple[str | None, str | None]: """Reasoning reaches `/api/generate` either in the top-level `thinking` field or inline in `` tags, never both. The field wins, matching `ollama_chat`.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( parse_content_for_reasoning, ) diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 0479ac5c36c..044c0131699 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -21,20 +21,24 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo should_convert_tool_call_to_json_mode, ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( - chat_completion_prompt_cache_breakpoints_for_wire, - drop_non_python_regex_patterns, drop_tool_reference_parts_from_tool_messages, - flatten_combinators_and_drop_non_python_regex_patterns, get_tool_call_names, hoist_images_from_tool_messages, system_messages_first, - tool_with_sanitized_parameters, - without_prompt_cache_breakpoints, ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_url_to_base64, convert_url_to_base64, ) +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + flatten_combinators_and_drop_non_python_regex_patterns, + tool_with_sanitized_parameters, +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + chat_completion_prompt_cache_breakpoints_for_wire, + without_prompt_cache_breakpoints, +) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.base_utils import ( diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 3cb798b7318..3d0abb1a375 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -21,10 +21,12 @@ from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( safe_convert_created_field, ) -from litellm.litellm_core_utils.prompt_templates.common_utils import ( - cache_control_as_prompt_cache_breakpoint, +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( drop_non_python_regex_patterns, flatten_combinators_and_drop_non_python_regex_patterns, +) +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + cache_control_as_prompt_cache_breakpoint, prompt_cache_breakpoint_for_wire, whole_prompt_cache_breakpoint_or_none, with_prompt_cache_breakpoint, diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 9159c9a4070..158ed979bad 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -35,7 +35,7 @@ from litellm.litellm_core_utils.cloud_storage_security import ( ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, iter_items from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( extract_file_data, extract_file_metadata, ) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 3244e0ae15b..9999c75a91c 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -20,9 +20,6 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap -from litellm.litellm_core_utils.prompt_templates.common_utils import ( - get_image_mime_type_from_url, -) from litellm.litellm_core_utils.prompt_templates.factory import ( GeminiMediaSource, convert_to_anthropic_image_obj, @@ -33,6 +30,9 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( get_thought_signature_from_tool, response_schema_prompt, ) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_image_mime_type_from_url, +) from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, async_inline_remote_media from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels diff --git a/litellm/llms/xai/files/transformation.py b/litellm/llms/xai/files/transformation.py index 88f2fc51983..a4260ec7ff2 100644 --- a/litellm/llms/xai/files/transformation.py +++ b/litellm/llms/xai/files/transformation.py @@ -13,7 +13,9 @@ from openai.types.file_deleted import FileDeleted from pydantic import ConfigDict from typing_extensions import ReadOnly, TypedDict -from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, +) from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.transformation import BaseFilesConfig, LiteLLMLoggingObj diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 5c2f6bab650..ba595ad47ee 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -58,6 +58,8 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import ( from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, +) +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( split_media_for_tpm, ) from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index e613d744fb0..9c49c61291f 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -434,7 +434,7 @@ async def route_create_file( ) storage: Final = explicit_storage or (LITELLM_DB_STORAGE_BACKEND_NAME if executed_model is not None else None) if storage is not None: - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( extract_file_data, ) from litellm.proxy.openai_files_endpoints.storage_backend_service import ( diff --git a/litellm/responses/litellm_completion_transformation/count_request.py b/litellm/responses/litellm_completion_transformation/count_request.py index 0b769326012..3deceac07aa 100644 --- a/litellm/responses/litellm_completion_transformation/count_request.py +++ b/litellm/responses/litellm_completion_transformation/count_request.py @@ -6,7 +6,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, update_responses_tools_with_model_file_ids, ) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index e9f9eb85d25..ae7c3fccffa 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -38,7 +38,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import ( ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, update_responses_tools_with_model_file_ids, ) diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index f3425596afa..062886b44fd 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -51,7 +51,7 @@ from litellm.exceptions import AuthenticationError, BadRequestError from litellm.integrations.custom_logger import CustomLogger, Span from litellm.litellm_core_utils.get_llm_provider_logic import declared_provider, get_llm_provider from litellm.litellm_core_utils.litellm_logging import has_sensitive_keyword -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( anthropic_content_lists, encrypted_content_of_block, strip_encrypted_reasoning_from_messages, diff --git a/litellm/rust_bridge/responses/detectors.py b/litellm/rust_bridge/responses/detectors.py index 763b63d6742..2fdb2c44b54 100644 --- a/litellm/rust_bridge/responses/detectors.py +++ b/litellm/rust_bridge/responses/detectors.py @@ -67,7 +67,7 @@ def _tools(request: NativeCall) -> Sequence[object]: def managed_resource_ids(request: NativeCall) -> bool: """Python replaces managed file IDs in Responses input and tools with provider IDs.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, update_responses_tools_with_model_file_ids, ) diff --git a/tests/unit/enterprise/proxy/hooks/test_managed_files.py b/tests/unit/enterprise/proxy/hooks/test_managed_files.py index 1755ab97689..a8dc506f306 100644 --- a/tests/unit/enterprise/proxy/hooks/test_managed_files.py +++ b/tests/unit/enterprise/proxy/hooks/test_managed_files.py @@ -1,8 +1,8 @@ +import asyncio import base64 import json import logging import time -import asyncio import uuid from collections.abc import Awaitable, Callable, Mapping from types import SimpleNamespace @@ -23,12 +23,13 @@ from litellm_enterprise.proxy.hooks.managed_files import ( PROXY_LiteLLMManagedFiles, _provider_file_retrieve_credentials, ) + from litellm.caching import DualCache from litellm.proxy._types import CallTypes, LiteLLM_ManagedFileTable, UserAPIKeyAuth from litellm.proxy.openai_files_endpoints.common_utils import ( BATCH_CREATE_HIDDEN_PARAM, - is_base64_encoded_unified_file_id, encode_file_id_with_model, + is_base64_encoded_unified_file_id, ) from litellm.types.llms.openai import OpenAIFileObject from tests.unit.litellm_core_utils.attribute_forwarder import AttributeForwarder @@ -209,6 +210,7 @@ def _marked_fallback_file_row( def _batch_response_for_output_listing() -> "LiteLLMBatch": from openai.types.batch import BatchRequestCounts + from litellm.types.utils import LiteLLMBatch response: Final = LiteLLMBatch( @@ -753,9 +755,9 @@ async def test_router_acreate_batch_only_selects_from_file_id_mapping(monkeypatc @pytest.mark.asyncio async def test_output_file_id_for_batch_retrieve(): - import litellm.proxy.proxy_server as proxy_server_module from openai.types.batch import BatchRequestCounts + import litellm.proxy.proxy_server as proxy_server_module from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import OpenAIFileObject from litellm.types.utils import LiteLLMBatch @@ -837,9 +839,9 @@ async def test_output_file_id_preserves_target_model_names_when_model_name_missi (e.g. Vertex batch retrieve), unified output_file_id should still include target_model_names from the managed input file ID. """ - import litellm.proxy.proxy_server as proxy_server_module from openai.types.batch import BatchRequestCounts + import litellm.proxy.proxy_server as proxy_server_module from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import OpenAIFileObject from litellm.types.utils import LiteLLMBatch @@ -925,9 +927,9 @@ async def test_error_file_id_for_failed_batch(): """ Test that the error_file_id is properly managed when a batch fails """ - import litellm.proxy.proxy_server as proxy_server_module from openai.types.batch import BatchRequestCounts + import litellm.proxy.proxy_server as proxy_server_module from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import OpenAIFileObject from litellm.types.utils import LiteLLMBatch @@ -1102,7 +1104,7 @@ def test_update_responses_input_with_unified_file_id(): Test that update_responses_input_with_model_file_ids correctly decodes unified file IDs and extracts llm_output_file_id from responses API input. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, ) @@ -1145,7 +1147,7 @@ def test_update_responses_input_with_regular_file_id(): Test that update_responses_input_with_model_file_ids keeps regular OpenAI file IDs unchanged. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, ) @@ -1181,7 +1183,7 @@ def test_update_responses_input_with_string_input(): """ Test that update_responses_input_with_model_file_ids returns string input unchanged. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, ) @@ -1198,7 +1200,7 @@ def test_update_responses_input_with_multiple_file_ids(): Test that update_responses_input_with_model_file_ids handles multiple file IDs (both unified and regular) in the same input. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, ) @@ -1242,7 +1244,7 @@ def test_update_responses_input_with_model_file_id_mapping(): Test that update_responses_input_with_model_file_ids correctly uses model_file_id_mapping to map managed file IDs to provider-specific file IDs. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_input_with_model_file_ids, ) @@ -1302,7 +1304,7 @@ def test_update_responses_tools_with_model_file_id_mapping(): tools.container.file_ids were not being replaced with provider-specific file IDs, causing "string too long" errors from OpenAI. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_tools_with_model_file_ids, ) @@ -1350,7 +1352,7 @@ def test_update_responses_tools_without_mapping(): Test that update_responses_tools_with_model_file_ids keeps file IDs unchanged when no mapping is provided. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_tools_with_model_file_ids, ) @@ -1382,7 +1384,7 @@ def test_update_responses_tools_with_mixed_file_ids(): Test that update_responses_tools_with_model_file_ids correctly handles a mix of managed and regular file IDs. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_responses_tools_with_model_file_ids, ) @@ -3943,7 +3945,6 @@ async def test_afile_retrieve_raises_error_for_non_managed_file(): @pytest.mark.asyncio async def test_list_batches_from_managed_objects_table(): - from openai.types.batch import BatchRequestCounts from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_file_id_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_file_id_utils.py new file mode 100644 index 00000000000..658e3f0d3cb --- /dev/null +++ b/tests/unit/litellm_core_utils/prompt_templates/test_file_id_utils.py @@ -0,0 +1,436 @@ +import os + +import pytest + +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_file_ids_from_messages, + get_format_from_file_id, + update_messages_with_model_file_ids, +) + + +UNIFIED_FILE_ID_B64 = ( + "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9wZGY7dW5pZmllZF9pZCw2YzBiNTg5MC04OTE0" + "LTQ4ZTAtYjhmNC0wYWU1ZWQzYzE0YTU7dGFyZ2V0X21vZGVsX25hbWVzLGdwdC00bztsbG1f" + "b3V0cHV0X2ZpbGVfaWQsZmlsZS1FQ0JQVzdNTDlnN1hIZHdHZ1VQWmFNO2xsbV9vdXRwdXRf" + "ZmlsZV9tb2RlbF9pZCxlMjY0NTNmOWU3NmU3OTkzNjgwZDAwNjhkOThjMWY0Y2MyMDViYmFk" + "MDk2N2EzM2M2NjQ4OTM1NjhjYTc0M2My" +) + + +def test_get_format_from_file_id(): + unified_file_id = "litellm_proxy:application/pdf;unified_id,cbbe3534-8bf8-4386-af00-f5f6b7e370bf" + + format = get_format_from_file_id(unified_file_id) + + assert format == "application/pdf" + + +def test_update_messages_with_model_file_ids(): + file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9wZGY7dW5pZmllZF9pZCxmYzdmMmVhNS0wZjUwLTQ5ZjYtODljMS03ZTZhNTRiMTIxMzg" + model_id = "my_model_id" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this recording?"}, + { + "type": "file", + "file": { + "file_id": file_id, + }, + }, + ], + }, + ] + + model_file_id_mapping = {file_id: {"my_model_id": "provider_file_id"}} + + updated_messages = update_messages_with_model_file_ids(messages, model_id, model_file_id_mapping) + + assert updated_messages == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this recording?"}, + { + "type": "file", + "file": { + "file_id": "provider_file_id", + "format": "application/pdf", + }, + }, + ], + } + ] + + +def test_get_file_ids_from_messages_skips_langchain_v1_file_block(): + """A LangChain v1 standardized file block must not crash file-id discovery.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarise this PDF"}, + # LangChain v1 shape produced by `_normalize_messages`. + # No `file` sub-dict: the discriminator is `type: "file"` but + # the payload lives on `base64`/`mime_type` siblings. + { + "type": "file", + "id": "lc_1", + "base64": "JVBERi0xLjQK", + "mime_type": "application/pdf", + "extras": {"file_format": "application/pdf"}, + }, + ], + } + ] + + assert get_file_ids_from_messages(messages) == [] + + +def test_get_file_ids_from_messages_still_extracts_from_openai_shape(): + """Well-formed OpenAI file blocks still yield their file_id.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is this?"}, + {"type": "file", "file": {"file_id": "file-abc"}}, + ], + } + ] + + assert get_file_ids_from_messages(messages) == ["file-abc"] + + +def test_get_file_ids_from_messages_mixed_shapes(): + """Mixed OpenAI and non-OpenAI file blocks: extract from the former, + ignore the latter.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "file", "file": {"file_id": "file-keep"}}, + { + "type": "file", + "id": "lc_2", + "base64": "AAA", + "mime_type": "application/pdf", + }, + ], + } + ] + + assert get_file_ids_from_messages(messages) == ["file-keep"] + + +def test_get_file_ids_from_messages_file_field_not_dict(): + """`file` set to a non-dict value (e.g. stringified payload) must not crash.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "file", "file": "unexpectedly-a-string"}, + ], + } + ] + + assert get_file_ids_from_messages(messages) == [] + + +def test_get_file_ids_from_messages_skips_bare_string_content_items(): + messages = [ + { + "role": "user", + "content": [ + "what type of file is this?", + {"type": "file", "file": {"file_id": "file-abc"}}, + ], + } + ] + + assert get_file_ids_from_messages(messages) == ["file-abc"] + + +def test_update_messages_with_model_file_ids_skips_non_openai_file_blocks(): + """`update_messages_with_model_file_ids` is also called on user content + before provider dispatch. It must tolerate non-OpenAI file blocks the same + way.""" + langchain_v1_block = { + "type": "file", + "id": "lc_3", + "base64": "AAA", + "mime_type": "application/pdf", + } + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "hello"}, + langchain_v1_block, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "model-1", {}) + + # Messages pass through unchanged when there is no `file` sub-dict to remap. + assert updated == messages + + +def test_update_messages_with_model_file_ids_decodes_unified_id_when_mapping_empty(): + """When the mapping is empty (e.g. multi-replica cache miss), the function + must decode the base64-encoded unified file id and substitute the embedded + llm_output_file_id — mirroring the Responses-API sibling.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this recording?"}, + { + "type": "file", + "file": { + "file_id": UNIFIED_FILE_ID_B64, + "format": "audio/wav", + }, + }, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "any-model-id", {}) + + assert updated[0]["content"][1]["file"]["file_id"] == "file-ECBPW7ML9g7XHdwGgUPZaM" + # Customer-supplied format is preserved (this is the field whose absence + # the misleading error message used to complain about). + assert updated[0]["content"][1]["file"]["format"] == "audio/wav" + + +def test_update_messages_with_model_file_ids_mapping_takes_precedence_over_decode(): + """When both mapping and decode would resolve, the mapping must win + (preserves per-deployment routing precision).""" + mapping = {UNIFIED_FILE_ID_B64: {"model-A": "mapped-provider-file-id"}} + messages = [ + { + "role": "user", + "content": [ + { + "type": "file", + "file": { + "file_id": UNIFIED_FILE_ID_B64, + "format": "application/pdf", + }, + }, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "model-A", mapping) + + assert updated[0]["content"][0]["file"]["file_id"] == "mapped-provider-file-id" + + +def test_update_messages_with_model_file_ids_non_unified_passes_through(): + """A raw provider id (e.g. gs:// URI or a random string) must be left + untouched when the mapping doesn't resolve it. The decode fallback must + not corrupt non-unified ids.""" + raw_id = "gs://my-bucket/uploads/abc-123.wav" + messages = [ + { + "role": "user", + "content": [ + {"type": "file", "file": {"file_id": raw_id, "format": "audio/wav"}}, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "model-A", {}) + + assert updated[0]["content"][0]["file"]["file_id"] == raw_id + + +def test_update_messages_with_model_file_ids_mapping_miss_falls_back_to_decode(): + """A mapping that exists but doesn't contain this file_id should still + trigger the decode fallback — covers the case where the hook resolved + *some* ids but not this one.""" + other_id = "some-other-file-id" + mapping = {other_id: {"model-A": "other-provider-id"}} + messages = [ + { + "role": "user", + "content": [ + { + "type": "file", + "file": {"file_id": UNIFIED_FILE_ID_B64, "format": "audio/wav"}, + }, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "model-A", mapping) + + assert updated[0]["content"][0]["file"]["file_id"] == "file-ECBPW7ML9g7XHdwGgUPZaM" + + +def test_update_messages_with_model_file_ids_tolerates_non_dict_content_items(): + """Content list items aren't always dicts. text_completion forwards + token-ids (list of ints, or list of list of ints for batch) through + this path. The function must skip non-dict items instead of indexing + into them.""" + messages_token_ids = [{"role": "user", "content": [15496, 995]}] + messages_token_ids_batch = [{"role": "user", "content": [[15496, 995], [9906, 0]]}] + + # Both should pass through unchanged without raising. + assert update_messages_with_model_file_ids(messages_token_ids, "model-A", {}) == messages_token_ids + assert update_messages_with_model_file_ids(messages_token_ids_batch, "model-A", {}) == messages_token_ids_batch + + +class TestExtractFileDataBareStr: + """``extract_file_data`` used to accept bare ``str`` values and ``open()`` + them server-side. When the helper runs inside a proxy request handler the + value is attacker-controlled, so the open() call was a textbook arbitrary + local file read. Lock the new contract: bare ``str`` is rejected with a + clear migration message; ``pathlib.Path`` is still accepted for SDK + ergonomics because it's a Python-level type that HTTP form values can't + fabricate.""" + + def test_rejects_bare_str(self): + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, + ) + + with pytest.raises(ValueError, match="does not accept bare str inputs"): + extract_file_data("/etc/passwd") + + def test_accepts_pathlib_path(self): + import tempfile + from pathlib import Path + + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, + ) + + content = b"hello" + with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as f: + f.write(content) + tmp_path = Path(f.name) + + try: + extracted = extract_file_data(tmp_path) + assert extracted.get("content") == content + finally: + os.unlink(str(tmp_path)) + + def test_accepts_bytes(self): + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, + ) + + extracted = extract_file_data(b"raw bytes content") + assert extracted.get("content") == b"raw bytes content" + + def test_accepts_tuple(self): + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + extract_file_data, + ) + + extracted = extract_file_data(("foo.txt", b"raw bytes content")) + assert extracted.get("filename") == "foo.txt" + assert extracted.get("content") == b"raw bytes content" + + +def _xlitellm_encoded(raw_id: str, model: str) -> str: + from litellm.proxy.openai_files_endpoints.common_utils import ( + encode_file_id_with_model, + ) + + return encode_file_id_with_model(raw_id, model) + + +def test_update_messages_with_model_file_ids_decodes_xlitellm_encoded_id(): + """x-litellm-model upload returns `file-;model,)>`. + Without decoding, the encoded id leaks to upstream OpenAI and errors as + 'Files [...] were not found'. Decode it back to raw provider id.""" + raw_id = "file-ExTuCawUqxEMjVFK6xwR9B" + encoded_id = _xlitellm_encoded(raw_id, "gpt-5.1") + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this."}, + {"type": "file", "file": {"file_id": encoded_id}}, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "model-A", {}) + + assert updated[0]["content"][1]["file"]["file_id"] == raw_id + + +def test_update_responses_input_with_model_file_ids_decodes_xlitellm_encoded_id(): + """Same bug on /v1/responses path. Without decoding the encoded id (>64 + chars), OpenAI rejects with 'string too long. Expected ... maximum length + 64'.""" + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + update_responses_input_with_model_file_ids, + ) + + raw_id = "file-ExTuCawUqxEMjVFK6xwR9B" + encoded_id = _xlitellm_encoded(raw_id, "gpt-5.1") + input_items = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Summarize."}, + {"type": "input_file", "file_id": encoded_id}, + ], + } + ] + + updated = update_responses_input_with_model_file_ids(input_items) + + assert updated[0]["content"][1]["file_id"] == raw_id + + +@pytest.mark.parametrize("unusable_file_id", [[1], {"a": 1}]) +def test_update_responses_input_with_model_file_ids_keeps_a_non_string_file_id_as_sent(unusable_file_id): + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + update_responses_input_with_model_file_ids, + ) + + encoded_id = _xlitellm_encoded("file-ExTuCawUqxEMjVFK6xwR9B", "gpt-5.1") + input_items = [ + { + "role": "user", + "content": [ + {"type": "input_file", "file_id": encoded_id}, + {"type": "input_file", "file_id": unusable_file_id}, + ], + } + ] + + updated = update_responses_input_with_model_file_ids(input_items, "model-A", {encoded_id: {"model-A": "file-on-a"}}) + + assert [item["file_id"] for item in updated[0]["content"]] == ["file-on-a", unusable_file_id] + + +def test_update_messages_xlitellm_decode_does_not_override_mapping(): + """If the call-site already resolved a provider id via the mapping, that + wins. The new decode fallback runs only when no mapping match.""" + raw_id = "file-ExTuCawUqxEMjVFK6xwR9B" + encoded_id = _xlitellm_encoded(raw_id, "gpt-5.1") + mapping = {encoded_id: {"model-A": "provider-explicit-id"}} + messages = [ + { + "role": "user", + "content": [ + {"type": "file", "file": {"file_id": encoded_id}}, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "model-A", mapping) + + assert updated[0]["content"][0]["file"]["file_id"] == "provider-explicit-id" diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_json_schema_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_json_schema_utils.py new file mode 100644 index 00000000000..19951c7efad --- /dev/null +++ b/tests/unit/litellm_core_utils/prompt_templates/test_json_schema_utils.py @@ -0,0 +1,801 @@ +import functools +import json +import sys +from collections.abc import Mapping +from typing import Final + +import pytest + +from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + schema_grammar_branch_count, + schema_has_open_object, + schema_has_unresolvable_ref, + schema_property_count, +) + +_ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$' +_ARTIFACT_DATA_ID_PATTERN: Final = r"^(?!\.\.?(?:\/|$))[A-Za-z0-9_\-.~:@+]{1,200}$" + +class TestFlattenTopLevelSchemaCombinators: + def _customer_anyof_schema(self): + return { + "type": "object", + "anyOf": [ + { + "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, + "required": ["id", "enabled"], + }, + { + "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, + "required": ["id", "schedule"], + }, + ], + "properties": {"id": {"type": "string"}}, + "required": ["id"], + } + + def test_merges_anyof_branches_into_object_schema(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + result = flatten_top_level_schema_combinators(self._customer_anyof_schema()) + + assert "anyOf" not in result + assert result["type"] == "object" + assert set(result["properties"]) == {"id", "enabled", "schedule"} + assert result["properties"]["enabled"] == {"type": "boolean"} + assert result["required"] == ["id"] + + def test_typeless_anyof_of_object_branches_gets_intersected_required(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "anyOf": [ + { + "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, + "required": ["id", "enabled"], + }, + { + "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, + "required": ["id", "schedule"], + }, + ] + } + + result = flatten_top_level_schema_combinators(schema) + + assert result["type"] == "object" + assert "anyOf" not in result + assert result["required"] == ["id"] + + def test_allof_required_is_the_union_of_branches(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "type": "object", + "allOf": [ + {"properties": {"id": {"type": "string"}}, "required": ["id"]}, + {"properties": {"enabled": {"type": "boolean"}}, "required": ["enabled"]}, + ], + } + + result = flatten_top_level_schema_combinators(schema) + + assert "allOf" not in result + assert result["required"] == ["enabled", "id"] + assert set(result["properties"]) == {"id", "enabled"} + + def test_top_level_schema_wins_property_collisions(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "type": "object", + "anyOf": [ + {"properties": {"id": {"type": "integer"}}}, + {"properties": {"id": {"type": "number"}}}, + ], + "properties": {"id": {"type": "string"}}, + } + + result = flatten_top_level_schema_combinators(schema) + + assert result["properties"]["id"] == {"type": "string"} + + def test_drops_openai_rejected_scalar_keys_on_object_schema(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "type": "object", + "properties": {"id": {"type": "string"}}, + "enum": [{"id": "a"}], + "const": {"id": "a"}, + "not": {"required": ["other"]}, + } + + result = flatten_top_level_schema_combinators(schema) + + assert "enum" not in result + assert "const" not in result + assert "not" not in result + assert result["properties"] == {"id": {"type": "string"}} + + def test_resolves_local_ref_branches_from_defs(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "anyOf": [{"$ref": "#/$defs/Enable"}, {"$ref": "#/$defs/Schedule"}], + "$defs": { + "Enable": { + "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, + "required": ["id", "enabled"], + }, + "Schedule": { + "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, + "required": ["id", "schedule"], + }, + }, + } + + result = flatten_top_level_schema_combinators(schema) + + assert "anyOf" not in result + assert result["type"] == "object" + assert set(result["properties"]) == {"id", "enabled", "schedule"} + assert result["required"] == ["id"] + assert "$defs" in result + + def test_flattens_nested_combinator_branch_from_definitions(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "type": "object", + "oneOf": [ + {"$ref": "#/definitions/Toggle"}, + {"allOf": [{"properties": {"schedule": {"type": "string"}}, "required": ["schedule"]}]}, + ], + "definitions": {"Toggle": {"properties": {"enabled": {"type": "boolean"}}, "required": ["enabled"]}}, + } + + result = flatten_top_level_schema_combinators(schema) + + assert "oneOf" not in result + assert set(result["properties"]) == {"enabled", "schedule"} + assert "required" not in result + + def test_unresolvable_ref_branch_leaves_schema_untouched(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "type": "object", + "anyOf": [{"$ref": "https://example.com/schemas/automation.json"}], + "properties": {"id": {"type": "string"}}, + } + + assert flatten_top_level_schema_combinators(schema) is schema + + def test_self_referencing_ref_branch_leaves_schema_untouched(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "type": "object", + "anyOf": [{"$ref": "#/$defs/Node"}], + "$defs": {"Node": {"type": "object", "anyOf": [{"$ref": "#/$defs/Node"}]}}, + } + + assert flatten_top_level_schema_combinators(schema) is schema + + @pytest.mark.parametrize("boolean_branch", [True, False]) + def test_boolean_branch_leaves_schema_untouched(self, boolean_branch): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "type": "object", + "anyOf": [boolean_branch, {"properties": {"id": {"type": "string"}}, "required": ["id"]}], + } + + assert flatten_top_level_schema_combinators(schema) is schema + + def test_root_required_is_combined_with_branch_required(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + allof_schema = { + "type": "object", + "required": ["id"], + "properties": {"id": {"type": "string"}}, + "allOf": [{"properties": {"enabled": {"type": "boolean"}}, "required": ["enabled"]}], + } + anyof_schema = { + "type": "object", + "required": ["id"], + "properties": {"id": {"type": "string"}}, + "anyOf": [ + {"properties": {"name": {"type": "string"}, "a": {"type": "string"}}, "required": ["name", "a"]}, + {"properties": {"name": {"type": "string"}, "b": {"type": "string"}}, "required": ["name", "b"]}, + ], + } + + assert flatten_top_level_schema_combinators(allof_schema)["required"] == ["enabled", "id"] + assert flatten_top_level_schema_combinators(anyof_schema)["required"] == ["id", "name"] + + def test_repeated_refs_are_expanded_once(self): + import time + + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + fan_out, chain_length = 8, 8 + schema = { + "type": "object", + "anyOf": [{"$ref": "#/$defs/Level0"}], + "$defs": { + **{ + f"Level{level}": {"anyOf": [{"$ref": f"#/$defs/Level{level + 1}"}] * fan_out} + for level in range(chain_length) + }, + f"Level{chain_length}": {"type": "object", "properties": {"id": {"type": "string"}}}, + }, + } + + started = time.perf_counter() + result = flatten_top_level_schema_combinators(schema) + + assert time.perf_counter() - started < 5 + assert "anyOf" not in result + assert result["properties"] == {"id": {"type": "string"}} + + def test_nesting_past_the_depth_cap_leaves_schema_untouched(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + def nested(levels): + leaf = {"type": "object", "properties": {"id": {"type": "string"}}} + return functools.reduce(lambda inner, _: {"type": "object", "anyOf": [inner]}, range(levels), leaf) + + shallow, deep = nested(20), nested(40) + + assert "anyOf" not in flatten_top_level_schema_combinators(shallow) + assert flatten_top_level_schema_combinators(deep) is deep + + @pytest.mark.parametrize( + "branches", + [ + [{"required": ["enabled"]}, {"required": ["schedule"]}], + [{"type": "object", "required": ["enabled"]}, {"type": "object", "required": ["schedule"]}], + ], + ) + def test_typeless_root_with_properties_flattens_branches_without_properties(self, branches): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}, "schedule": {"type": "string"}}, + "required": ["id"], + "anyOf": branches, + } + + result = flatten_top_level_schema_combinators(schema) + + assert "anyOf" not in result + assert result["type"] == "object" + assert set(result["properties"]) == {"id", "enabled", "schedule"} + assert result["required"] == ["id"] + + def test_typeless_root_flattens_typed_object_branches_without_properties(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = { + "anyOf": [ + {"type": "object", "properties": {"id": {"type": "string"}}}, + {"type": "object", "required": ["id"]}, + ] + } + + result = flatten_top_level_schema_combinators(schema) + + assert "anyOf" not in result + assert result["type"] == "object" + assert result["properties"] == {"id": {"type": "string"}} + assert "required" not in result + + def test_non_object_union_passes_through_unchanged(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = {"anyOf": [{"type": "string"}, {"type": "number"}]} + + assert flatten_top_level_schema_combinators(schema) is schema + + def test_schema_without_rejected_keys_is_returned_as_is(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = {"type": "object", "properties": {"nested": {"anyOf": [{"type": "string"}, {"type": "null"}]}}} + + assert flatten_top_level_schema_combinators(schema) is schema + + def test_input_schema_is_never_mutated(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_top_level_schema_combinators, + ) + + schema = self._customer_anyof_schema() + snapshot = json.loads(json.dumps(schema)) + + flatten_top_level_schema_combinators(schema) + + assert schema == snapshot + + +class TestToolWithSanitizedParameters: + def _anyof_tool(self): + return { + "type": "function", + "function": { + "name": "automation_update", + "description": "Update an automation", + "parameters": { + "type": "object", + "anyOf": [ + { + "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, + "required": ["id", "enabled"], + }, + { + "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, + "required": ["id", "schedule"], + }, + ], + "properties": {"id": {"type": "string"}}, + "required": ["id"], + }, + }, + } + + def test_flattens_anyof_parameters_into_new_tool(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_combinators_and_drop_non_python_regex_patterns, + tool_with_sanitized_parameters, + ) + + tool = self._anyof_tool() + result = tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) + + assert result is not tool + parameters = result["function"]["parameters"] + assert "anyOf" not in parameters + assert parameters["type"] == "object" + assert set(parameters["properties"]) == {"id", "enabled", "schedule"} + assert parameters["required"] == ["id"] + assert result["function"]["name"] == "automation_update" + assert tool == self._anyof_tool() + + def test_clean_parameters_return_the_same_tool_object(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_combinators_and_drop_non_python_regex_patterns, + tool_with_sanitized_parameters, + ) + + tool = { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}, + }, + } + + assert tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) is tool + + def test_pattern_only_sanitizer_drops_the_regex_and_keeps_the_union(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + tool_with_sanitized_parameters, + ) + + tool = self._anyof_tool() + tool["function"]["parameters"]["properties"]["id"]["pattern"] = _ARTIFACT_FIELD_PATTERN + + result = tool_with_sanitized_parameters(tool, drop_non_python_regex_patterns) + + parameters = result["function"]["parameters"] + assert parameters["properties"]["id"] == {"type": "string"} + assert parameters["anyOf"] == self._anyof_tool()["function"]["parameters"]["anyOf"] + assert tool["function"]["parameters"]["properties"]["id"]["pattern"] == _ARTIFACT_FIELD_PATTERN + + @pytest.mark.parametrize( + "tool", + [ + {"type": "function"}, + {"type": "function", "function": "not-a-dict"}, + {"type": "function", "function": {"name": "no_params"}}, + {"type": "function", "function": {"name": "bad_params", "parameters": "not-a-dict"}}, + ], + ) + def test_non_dict_function_or_parameters_return_the_same_tool_object(self, tool): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + flatten_combinators_and_drop_non_python_regex_patterns, + tool_with_sanitized_parameters, + ) + + assert tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) is tool + + def test_sanitizes_the_input_schema_of_an_anthropic_tool(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_lookaround_regex_patterns, + tool_with_sanitized_parameters, + ) + + tool = { + "name": "ArtifactData", + "description": "Read a shared database", + "input_schema": { + "type": "object", + "properties": {"doc_id": {"type": "string", "pattern": _ARTIFACT_DATA_ID_PATTERN}}, + }, + } + + result = tool_with_sanitized_parameters(tool, drop_lookaround_regex_patterns) + + assert result == { + "name": "ArtifactData", + "description": "Read a shared database", + "input_schema": {"type": "object", "properties": {"doc_id": {"type": "string"}}}, + } + assert tool["input_schema"]["properties"]["doc_id"]["pattern"] == _ARTIFACT_DATA_ID_PATTERN + + def test_returns_the_same_anthropic_tool_when_its_schema_has_nothing_to_drop(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_lookaround_regex_patterns, + tool_with_sanitized_parameters, + ) + + tool = {"name": "Read", "input_schema": {"type": "object", "properties": {"path": {"type": "string"}}}} + + assert tool_with_sanitized_parameters(tool, drop_lookaround_regex_patterns) is tool + + +def _regex_schema(pattern): + return { + "type": "object", + "properties": { + "field": {"type": "string", "pattern": pattern}, + "writes": { + "type": "array", + "items": {"properties": {"doc_id": {"type": "string", "pattern": pattern}}}, + }, + "query": {"anyOf": [{"type": "string", "pattern": pattern}, {"type": "null"}]}, + "pair": {"type": "array", "prefixItems": [{"type": "string", "pattern": pattern}]}, + "extra": {"type": "object", "additionalProperties": {"type": "string", "pattern": pattern}}, + "tagged": { + "type": "object", + "patternProperties": {pattern: {"type": "string"}, "^x_": {"type": "integer"}}, + }, + }, + "$defs": {"segment": {"type": "string", "pattern": pattern}}, + "required": ["field"], + } + + +class TestDropNonPythonRegexPatterns: + """Claude Code's Artifact tool declares ECMA-262 ``\\p{..}`` escapes that OpenAI's + validator, which compiles ``pattern`` values and ``patternProperties`` keys with + Python ``re``, refuses as "not a 'regex'".""" + + def test_drops_every_regex_python_re_rejects_from_every_schema_position(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + ) + + schema = _regex_schema(_ARTIFACT_FIELD_PATTERN) + + result = drop_non_python_regex_patterns(schema) + + assert '"pattern"' not in json.dumps(result) + properties = result["properties"] + assert properties["field"] == {"type": "string"} + assert properties["writes"]["items"]["properties"]["doc_id"] == {"type": "string"} + assert properties["query"]["anyOf"] == [{"type": "string"}, {"type": "null"}] + assert properties["pair"]["prefixItems"] == [{"type": "string"}] + assert properties["extra"]["additionalProperties"] == {"type": "string"} + assert properties["tagged"]["patternProperties"] == {"^x_": {"type": "integer"}} + assert result["$defs"]["segment"] == {"type": "string"} + assert result["required"] == ["field"] + assert schema == _regex_schema(_ARTIFACT_FIELD_PATTERN) + + def test_keeps_regexes_python_re_compiles_and_returns_the_same_object(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + ) + + schema = _regex_schema(r'^(?!__.*__$)[^"\\./[\]]{1,200}$') + + assert drop_non_python_regex_patterns(schema) is schema + + def test_pattern_keys_inside_data_positions_are_not_regexes(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + ) + + schema = { + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "template": {"type": "object", "default": {"pattern": _ARTIFACT_FIELD_PATTERN}}, + "samples": {"type": "array", "examples": [{"pattern": _ARTIFACT_FIELD_PATTERN}]}, + "fixed": {"const": {"pattern": _ARTIFACT_FIELD_PATTERN}}, + "vendor": {"type": "string", "x-litellm": {"pattern": _ARTIFACT_FIELD_PATTERN}}, + }, + "required": ["pattern"], + } + + assert drop_non_python_regex_patterns(schema) is schema + + def test_regex_nested_past_what_python_re_can_parse_is_dropped_not_raised(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + ) + + schema = { + "type": "object", + "properties": {"deep": {"type": "string", "pattern": "(" * 2000 + "a" + ")" * 2000}}, + } + + assert drop_non_python_regex_patterns(schema)["properties"]["deep"] == {"type": "string"} + + def test_walks_schemas_deeper_than_the_interpreter_recursion_limit(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + ) + + depth = sys.getrecursionlimit() + leaf = {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN} + schema = functools.reduce( + lambda inner, _: {"type": "object", "properties": {"child": inner}}, range(depth), leaf + ) + + result = drop_non_python_regex_patterns(schema) + + assert functools.reduce(lambda node, _: node["properties"]["child"], range(depth), result) == {"type": "string"} + assert functools.reduce(lambda node, _: node["properties"]["child"], range(depth), schema) is leaf + + def test_leaves_levels_past_the_json_nesting_limit_alone(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_non_python_regex_patterns, + ) + + leaf = {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN} + schema = functools.reduce( + lambda inner, _: {"type": "object", "properties": {"child": inner}}, range(1100), leaf + ) + + assert drop_non_python_regex_patterns(schema) is schema + + +class TestDropLookaroundRegexPatterns: + """Kimi K3 and Grok 4.6/4.7 on Bedrock Converse reject every tool schema regex that + uses a lookaround assertion, Claude Code's ``ArtifactData`` ``pattern`` included.""" + + @pytest.mark.parametrize( + "pattern", + [r"^(?!x).*$", r"^(?=.*a).*$", r"^.*(?\w+)$", r"^(?i)abc$", r"^[^\p{Cc}\p{Cf}]{1,200}$"], + ids=["plain", "non-capturing-group", "named-group", "inline-flag", "non-python-without-lookaround"], + ) + def test_keeps_regexes_without_lookaround_and_returns_the_same_object(self, pattern): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_lookaround_regex_patterns, + ) + + schema = _regex_schema(pattern) + + assert drop_lookaround_regex_patterns(schema) is schema + + def test_lookaround_inside_data_positions_is_not_a_regex(self): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_lookaround_regex_patterns, + ) + + schema = { + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "template": {"type": "object", "default": {"pattern": _ARTIFACT_DATA_ID_PATTERN}}, + "hint": {"type": "string", "description": "ids match " + _ARTIFACT_DATA_ID_PATTERN}, + }, + "required": ["pattern"], + } + + assert drop_lookaround_regex_patterns(schema) is schema + + +@pytest.mark.parametrize( + ("dropper", "patterns"), + [ + ("drop_non_python_regex_patterns", (_ARTIFACT_FIELD_PATTERN, r"^\p{L}+$")), + ("drop_lookaround_regex_patterns", (_ARTIFACT_DATA_ID_PATTERN, r"^(?=.*[a-z])\w+$")), + ], + ids=["non-python", "lookaround"], +) +class TestDroppedPatternPropertiesKeepTheirNamesAllowed: + """Dropping a ``patternProperties`` key from an object closed by ``additionalProperties: + false`` must not ban the names that key allowed: its value schema takes over as the + object's ``additionalProperties``.""" + + @staticmethod + def _drop(dropper): + from litellm.litellm_core_utils.prompt_templates.json_schema_utils import ( + drop_lookaround_regex_patterns, + drop_non_python_regex_patterns, + ) + + return { + "drop_non_python_regex_patterns": drop_non_python_regex_patterns, + "drop_lookaround_regex_patterns": drop_lookaround_regex_patterns, + }[dropper] + + def test_closed_object_takes_the_dropped_value_schema(self, dropper, patterns): + schema = { + "type": "object", + "patternProperties": {patterns[0]: {"type": "string", "pattern": patterns[0]}}, + "additionalProperties": False, + } + + assert self._drop(dropper)(schema) == { + "type": "object", + "patternProperties": {}, + "additionalProperties": {"type": "string"}, + } + + def test_closed_object_losing_two_entries_accepts_either_value_schema(self, dropper, patterns): + schema = { + "type": "object", + "patternProperties": { + patterns[0]: {"type": "string"}, + patterns[1]: {"type": "integer"}, + "^x_": {"type": "boolean"}, + }, + "additionalProperties": False, + } + + assert self._drop(dropper)(schema) == { + "type": "object", + "patternProperties": {"^x_": {"type": "boolean"}}, + "additionalProperties": {"anyOf": [{"type": "string"}, {"type": "integer"}]}, + } + + def test_object_with_its_own_additional_properties_schema_keeps_it(self, dropper, patterns): + schema = { + "type": "object", + "patternProperties": {patterns[0]: {"type": "string"}}, + "additionalProperties": {"type": "integer"}, + } + + assert self._drop(dropper)(schema) == { + "type": "object", + "patternProperties": {}, + "additionalProperties": {"type": "integer"}, + } + + +@pytest.mark.parametrize( + ("schema", "is_open"), + [ + ({"type": "object", "properties": {"a": {"type": "string"}}}, False), + ({"type": "object", "properties": {}}, False), + ({"type": "object", "additionalProperties": False}, False), + ({"type": "string"}, False), + ({"type": "object"}, True), + ({"type": ["object", "null"]}, True), + ({"type": "object", "properties": {"a": {"type": "string"}}, "additionalProperties": True}, True), + ({"type": "object", "additionalProperties": {"type": "integer"}}, True), + ({"type": "object", "properties": {}, "patternProperties": {"^x": {"type": "string"}}}, True), + ({"type": "object", "properties": {"inner": {"anyOf": [{"type": "object"}, {"type": "null"}]}}}, True), + ({"type": "array", "items": {"type": "object", "additionalProperties": {"type": "string"}}}, True), + ], +) +def test_schema_has_open_object_flags_any_node_that_accepts_unlisted_keys(schema, is_open): + assert schema_has_open_object(schema) is is_open + + +def _object_with(properties: Mapping[str, object], **defs: Mapping[str, object]) -> Mapping[str, object]: + return {"type": "object", "properties": properties, **defs} + + +@pytest.mark.parametrize( + ("schema", "unresolvable"), + [ + (_object_with({"a": {"type": "string"}}), False), + (_object_with({"a": {"$ref": "#/$defs/A"}, "b": {"$ref": "#/$defs/A"}}, **{"$defs": {"A": {"type": "string"}}}), False), + (_object_with({"a": {"$ref": "#/definitions/A"}}, definitions={"A": {"$ref": "#/definitions/B"}, "B": {"type": "integer"}}), False), + (_object_with({"a": {"$ref": "/$defs/A"}}, **{"$defs": {"A": {"items": {"$ref": "/$defs/B"}}, "B": {"type": "string"}}}), False), + (_object_with({"a": {"$ref": "#/$defs/Missing"}}, **{"$defs": {"A": {"type": "string"}}}), True), + (_object_with({"a": {"$ref": "other.json#/A"}}), True), + (_object_with({"parent": {"$ref": "#"}}), True), + (_object_with({"a": {"$ref": "#/$defs/A"}}, **{"$defs": {"A": {"type": "array", "items": {"$ref": "#/$defs/A"}}}}), True), + (_object_with({"a": {"$ref": "/$defs/A"}}, **{"$defs": {"A": {"type": "array", "items": {"$ref": "/$defs/A"}}}}), True), + ( + _object_with( + {"a": {"$ref": "#/$defs/A"}}, + **{"$defs": {"A": {"properties": {"b": {"$ref": "#/$defs/B"}}}, "B": {"properties": {"a": {"$ref": "#/$defs/A"}}}}}, + ), + True, + ), + ], + ids=["no-refs", "shared-def", "chained-definitions", "root-path-defs", "missing-def", "external", "root", "self-cycle", "root-path-self-cycle", "mutual-cycle"], +) +def test_schema_has_unresolvable_ref_flags_refs_that_cannot_be_inlined(schema, unresolvable): + assert schema_has_unresolvable_ref(schema) is unresolvable + + +@pytest.mark.parametrize( + ("schema", "properties", "branches"), + [ + (_object_with({"a": {"type": "string"}, "b": {"type": "string"}, "c": {"type": "string"}}, required=["a"]), 3, 8), + ( + _object_with( + {"x": {"$ref": "#/$defs/Pair"}, "y": {"$ref": "#/$defs/Pair"}}, + required=["x", "y"], + **{"$defs": {"Pair": _object_with({"l": {"type": "string"}, "r": {"type": "string"}}, required=["l"])}}, + ), + 4, + 7, + ), + (_object_with({"inner": _object_with({"z": {"type": "string"}})}, required=["inner"]), 2, 4), + ], + ids=["flat", "shared-def-counted-once", "nested"], +) +def test_schema_size_counts_every_written_object_once(schema, properties, branches): + assert schema_property_count(schema) == properties + assert schema_grammar_branch_count(schema) == branches diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index a17deedf7a9..390da500c32 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -1,51 +1,24 @@ -import base64 import copy -import functools import json -import os -import sys from collections.abc import Mapping from typing import Final, cast import pytest from litellm.litellm_core_utils.prompt_templates.common_utils import ( - ENCRYPTED_REASONING_SIGNATURE_PREFIX, TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, TOOL_RESULT_MEDIA_BOUNDARY, TOOL_RESULT_MEDIA_PLACEHOLDER, add_system_prompt_to_messages, - cache_control_as_prompt_cache_breakpoint, - converse_redacted_content, - encrypted_content_from_signature, - encrypted_reasoning_signature, - get_file_ids_from_messages, - get_format_from_file_id, get_semantic_cache_prompt_from_messages, get_str_from_messages, handle_any_messages_to_chat_completion_str_messages_conversion, handle_messages_with_content_list_to_str_conversion, hoist_images_from_tool_messages, - is_encrypted_reasoning_block, merge_consecutive_system_messages, - native_redacted_thinking_data, - parse_tool_call_arguments, - prompt_cache_breakpoint_for_wire, - responses_reasoning_items_from_thinking_blocks, - salvage_concatenated_tool_arguments, - schema_grammar_branch_count, - schema_has_open_object, - schema_has_unresolvable_ref, - schema_property_count, - split_concatenated_json_objects, - split_embedding_media_for_tpm, - split_media_for_tpm, - strip_encrypted_reasoning_from_messages, strip_litellm_response_only_fields, system_messages_first, - update_messages_with_model_file_ids, - with_prompt_cache_breakpoint, ) from litellm.types.llms.openai import ChatCompletionAudioObject, ChatCompletionFileObject, ChatCompletionImageObject @@ -53,53 +26,6 @@ _ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\ _ARTIFACT_DATA_ID_PATTERN: Final = r"^(?!\.\.?(?:\/|$))[A-Za-z0-9_\-.~:@+]{1,200}$" -def test_get_format_from_file_id(): - unified_file_id = "litellm_proxy:application/pdf;unified_id,cbbe3534-8bf8-4386-af00-f5f6b7e370bf" - - format = get_format_from_file_id(unified_file_id) - - assert format == "application/pdf" - - -def test_update_messages_with_model_file_ids(): - file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9wZGY7dW5pZmllZF9pZCxmYzdmMmVhNS0wZjUwLTQ5ZjYtODljMS03ZTZhNTRiMTIxMzg" - model_id = "my_model_id" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is in this recording?"}, - { - "type": "file", - "file": { - "file_id": file_id, - }, - }, - ], - }, - ] - - model_file_id_mapping = {file_id: {"my_model_id": "provider_file_id"}} - - updated_messages = update_messages_with_model_file_ids(messages, model_id, model_file_id_mapping) - - assert updated_messages == [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is in this recording?"}, - { - "type": "file", - "file": { - "file_id": "provider_file_id", - "format": "application/pdf", - }, - }, - ], - } - ] - - def test_handle_any_messages_to_chat_completion_str_messages_conversion_list(): # Test with list of messages messages = [ @@ -218,413 +144,6 @@ def test_convert_prefix_message_to_non_prefix_messages(): # ── split_concatenated_json_objects tests ── -def test_split_concatenated_json_single_object(): - """A single valid JSON object is returned as a one-element list.""" - result = split_concatenated_json_objects('{"location": "Boston"}') - assert result == [{"location": "Boston"}] - - -def test_split_concatenated_json_multiple_objects(): - """ - Multiple JSON objects concatenated without separators are split correctly. - This is the exact pattern from issue #20543 where Bedrock Claude Sonnet 4.5 - returns concatenated JSON in a single tool call arguments string. - """ - raw = ( - '{"command": ["curl", "-i", "http://localhost:9009"]}' - '{"command": ["curl", "-i", "http://localhost:9009/robots.txt"]}' - '{"command": ["curl", "-i", "http://localhost:9009/sitemap.xml"]}' - ) - result = split_concatenated_json_objects(raw) - assert len(result) == 3 - assert result[0] == {"command": ["curl", "-i", "http://localhost:9009"]} - assert result[1] == {"command": ["curl", "-i", "http://localhost:9009/robots.txt"]} - assert result[2] == {"command": ["curl", "-i", "http://localhost:9009/sitemap.xml"]} - - -def test_split_concatenated_json_with_whitespace(): - """Objects separated by whitespace are handled correctly.""" - raw = '{"a": 1} {"b": 2}\n{"c": 3}' - result = split_concatenated_json_objects(raw) - assert len(result) == 3 - assert result[0] == {"a": 1} - assert result[1] == {"b": 2} - assert result[2] == {"c": 3} - - -def test_split_concatenated_json_empty_string(): - """Empty or whitespace-only strings return an empty list.""" - assert split_concatenated_json_objects("") == [] - assert split_concatenated_json_objects(" ") == [] - - -def test_split_concatenated_json_non_dict_value(): - """Non-dict JSON values (e.g. arrays, strings) are replaced with {}.""" - result = split_concatenated_json_objects("[1, 2, 3]") - assert result == [{}] - - -def test_split_concatenated_json_wholly_invalid_returns_empty(): - """ - Wholly unparseable JSON degrades to an empty list instead of raising. - - Regression for https://github.com/BerriAI/litellm/issues/18667: a raise - here propagated out of `_convert_to_bedrock_tool_call_invoke` and turned - every replayed conversation into a 500. - """ - assert split_concatenated_json_objects("not json at all") == [] - - -def test_split_concatenated_json_malformed_object_returns_empty(): - """ - A single malformed object (missing comma between keys) degrades to an - empty list rather than raising `Expecting ',' delimiter`. - """ - assert split_concatenated_json_objects('{"location": "Boston" "unit": "celsius"}') == [] - - -def test_split_concatenated_json_salvages_prefix_before_truncated_tail(): - """ - Complete objects parsed before an unparseable/truncated tail are kept; - only the bad tail is discarded. - """ - result = split_concatenated_json_objects('{"a": 1}{"b": 2}{"c":') - assert result == [{"a": 1}, {"b": 2}] - - -def test_parse_tool_call_arguments_rejects_concatenated_json() -> None: - with pytest.raises(ValueError, match="Failed to parse tool call arguments"): - parse_tool_call_arguments('{"a":1}{"b":2}') - - -def _distinct_json_objects(count: int) -> str: - return "".join(json.dumps({"n": index}, separators=(",", ":")) for index in range(count)) - - -@pytest.mark.parametrize( - ("raw", "expected"), - ( - ('{"a":1}{"b":2}', ({"a": 1}, {"b": 2})), - ('{"a":1}{"a":1}{"a":1}', ({"a": 1},)), - ('{"a":1}{"a":1}{"b":2}', ({"a": 1}, {"a": 1}, {"b": 2})), - (_distinct_json_objects(8), tuple({"n": index} for index in range(8))), - (_distinct_json_objects(9), ()), - (_distinct_json_objects(9) + " junk", ()), - ('{"a":1}' * 7 + '{"b":2}', tuple({"a": 1} for _ in range(7)) + ({"b": 2},)), - ('{"a":1}' * 8 + '{"b":2}', ()), - ('{"a":1}' * 5000, ({"a": 1},)), - ('{"a":1}' * 20, ({"a": 1},)), - ('{"a":1}{"b":', ()), - ('0{"x":1}', ()), - ('{"x":1}0', ()), - ('[1]{"x":1}', ()), - ('{"a":1}{"b":2}}', ()), - ('{"a":1} junk', ()), - ), -) -def test_salvage_concatenated_tool_arguments(raw: str, expected: tuple[dict[str, object], ...]) -> None: - assert salvage_concatenated_tool_arguments(raw) == expected - - -# --------------------------------------------------------------------------- -# Regression tests for non-OpenAI file content blocks. -# -# `type: "file"` is a public content-block discriminator. Several producers -# (LangChain v1, provider-native shapes, custom user code) emit blocks with -# `type: "file"` but without the OpenAI Chat Completions `file` sub-dict. -# The discovery helpers below are used unconditionally inside -# `AnthropicConfig.validate_environment`, so any crash there surfaces as a -# `500 InternalServerError` before the request is even dispatched. -# --------------------------------------------------------------------------- - - -def test_get_file_ids_from_messages_skips_langchain_v1_file_block(): - """A LangChain v1 standardized file block must not crash file-id discovery.""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "summarise this PDF"}, - # LangChain v1 shape produced by `_normalize_messages`. - # No `file` sub-dict: the discriminator is `type: "file"` but - # the payload lives on `base64`/`mime_type` siblings. - { - "type": "file", - "id": "lc_1", - "base64": "JVBERi0xLjQK", - "mime_type": "application/pdf", - "extras": {"file_format": "application/pdf"}, - }, - ], - } - ] - - assert get_file_ids_from_messages(messages) == [] - - -def test_get_file_ids_from_messages_still_extracts_from_openai_shape(): - """Well-formed OpenAI file blocks still yield their file_id.""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "what is this?"}, - {"type": "file", "file": {"file_id": "file-abc"}}, - ], - } - ] - - assert get_file_ids_from_messages(messages) == ["file-abc"] - - -def test_get_file_ids_from_messages_mixed_shapes(): - """Mixed OpenAI and non-OpenAI file blocks: extract from the former, - ignore the latter.""" - messages = [ - { - "role": "user", - "content": [ - {"type": "file", "file": {"file_id": "file-keep"}}, - { - "type": "file", - "id": "lc_2", - "base64": "AAA", - "mime_type": "application/pdf", - }, - ], - } - ] - - assert get_file_ids_from_messages(messages) == ["file-keep"] - - -def test_get_file_ids_from_messages_file_field_not_dict(): - """`file` set to a non-dict value (e.g. stringified payload) must not crash.""" - messages = [ - { - "role": "user", - "content": [ - {"type": "file", "file": "unexpectedly-a-string"}, - ], - } - ] - - assert get_file_ids_from_messages(messages) == [] - - -def test_get_file_ids_from_messages_skips_bare_string_content_items(): - messages = [ - { - "role": "user", - "content": [ - "what type of file is this?", - {"type": "file", "file": {"file_id": "file-abc"}}, - ], - } - ] - - assert get_file_ids_from_messages(messages) == ["file-abc"] - - -def test_update_messages_with_model_file_ids_skips_non_openai_file_blocks(): - """`update_messages_with_model_file_ids` is also called on user content - before provider dispatch. It must tolerate non-OpenAI file blocks the same - way.""" - langchain_v1_block = { - "type": "file", - "id": "lc_3", - "base64": "AAA", - "mime_type": "application/pdf", - } - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "hello"}, - langchain_v1_block, - ], - } - ] - - updated = update_messages_with_model_file_ids(messages, "model-1", {}) - - # Messages pass through unchanged when there is no `file` sub-dict to remap. - assert updated == messages - - -# Reusable fixture (decodes to: litellm_proxy:application/pdf;unified_id,...; -# target_model_names,gpt-4o;llm_output_file_id,file-ECBPW7ML9g7XHdwGgUPZaM; -# llm_output_file_model_id,...) -UNIFIED_FILE_ID_B64 = ( - "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9wZGY7dW5pZmllZF9pZCw2YzBiNTg5MC04OTE0" - "LTQ4ZTAtYjhmNC0wYWU1ZWQzYzE0YTU7dGFyZ2V0X21vZGVsX25hbWVzLGdwdC00bztsbG1f" - "b3V0cHV0X2ZpbGVfaWQsZmlsZS1FQ0JQVzdNTDlnN1hIZHdHZ1VQWmFNO2xsbV9vdXRwdXRf" - "ZmlsZV9tb2RlbF9pZCxlMjY0NTNmOWU3NmU3OTkzNjgwZDAwNjhkOThjMWY0Y2MyMDViYmFk" - "MDk2N2EzM2M2NjQ4OTM1NjhjYTc0M2My" -) - - -def test_update_messages_with_model_file_ids_decodes_unified_id_when_mapping_empty(): - """When the mapping is empty (e.g. multi-replica cache miss), the function - must decode the base64-encoded unified file id and substitute the embedded - llm_output_file_id — mirroring the Responses-API sibling.""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is in this recording?"}, - { - "type": "file", - "file": { - "file_id": UNIFIED_FILE_ID_B64, - "format": "audio/wav", - }, - }, - ], - } - ] - - updated = update_messages_with_model_file_ids(messages, "any-model-id", {}) - - assert updated[0]["content"][1]["file"]["file_id"] == "file-ECBPW7ML9g7XHdwGgUPZaM" - # Customer-supplied format is preserved (this is the field whose absence - # the misleading error message used to complain about). - assert updated[0]["content"][1]["file"]["format"] == "audio/wav" - - -def test_update_messages_with_model_file_ids_mapping_takes_precedence_over_decode(): - """When both mapping and decode would resolve, the mapping must win - (preserves per-deployment routing precision).""" - mapping = {UNIFIED_FILE_ID_B64: {"model-A": "mapped-provider-file-id"}} - messages = [ - { - "role": "user", - "content": [ - { - "type": "file", - "file": { - "file_id": UNIFIED_FILE_ID_B64, - "format": "application/pdf", - }, - }, - ], - } - ] - - updated = update_messages_with_model_file_ids(messages, "model-A", mapping) - - assert updated[0]["content"][0]["file"]["file_id"] == "mapped-provider-file-id" - - -def test_update_messages_with_model_file_ids_non_unified_passes_through(): - """A raw provider id (e.g. gs:// URI or a random string) must be left - untouched when the mapping doesn't resolve it. The decode fallback must - not corrupt non-unified ids.""" - raw_id = "gs://my-bucket/uploads/abc-123.wav" - messages = [ - { - "role": "user", - "content": [ - {"type": "file", "file": {"file_id": raw_id, "format": "audio/wav"}}, - ], - } - ] - - updated = update_messages_with_model_file_ids(messages, "model-A", {}) - - assert updated[0]["content"][0]["file"]["file_id"] == raw_id - - -def test_update_messages_with_model_file_ids_mapping_miss_falls_back_to_decode(): - """A mapping that exists but doesn't contain this file_id should still - trigger the decode fallback — covers the case where the hook resolved - *some* ids but not this one.""" - other_id = "some-other-file-id" - mapping = {other_id: {"model-A": "other-provider-id"}} - messages = [ - { - "role": "user", - "content": [ - { - "type": "file", - "file": {"file_id": UNIFIED_FILE_ID_B64, "format": "audio/wav"}, - }, - ], - } - ] - - updated = update_messages_with_model_file_ids(messages, "model-A", mapping) - - assert updated[0]["content"][0]["file"]["file_id"] == "file-ECBPW7ML9g7XHdwGgUPZaM" - - -def test_update_messages_with_model_file_ids_tolerates_non_dict_content_items(): - """Content list items aren't always dicts. text_completion forwards - token-ids (list of ints, or list of list of ints for batch) through - this path. The function must skip non-dict items instead of indexing - into them.""" - messages_token_ids = [{"role": "user", "content": [15496, 995]}] - messages_token_ids_batch = [{"role": "user", "content": [[15496, 995], [9906, 0]]}] - - # Both should pass through unchanged without raising. - assert update_messages_with_model_file_ids(messages_token_ids, "model-A", {}) == messages_token_ids - assert update_messages_with_model_file_ids(messages_token_ids_batch, "model-A", {}) == messages_token_ids_batch - - -class TestExtractFileDataBareStr: - """``extract_file_data`` used to accept bare ``str`` values and ``open()`` - them server-side. When the helper runs inside a proxy request handler the - value is attacker-controlled, so the open() call was a textbook arbitrary - local file read. Lock the new contract: bare ``str`` is rejected with a - clear migration message; ``pathlib.Path`` is still accepted for SDK - ergonomics because it's a Python-level type that HTTP form values can't - fabricate.""" - - def test_rejects_bare_str(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_file_data, - ) - - with pytest.raises(ValueError, match="does not accept bare str inputs"): - extract_file_data("/etc/passwd") - - def test_accepts_pathlib_path(self): - import tempfile - from pathlib import Path - - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_file_data, - ) - - content = b"hello" - with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as f: - f.write(content) - tmp_path = Path(f.name) - - try: - extracted = extract_file_data(tmp_path) - assert extracted.get("content") == content - finally: - os.unlink(str(tmp_path)) - - def test_accepts_bytes(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_file_data, - ) - - extracted = extract_file_data(b"raw bytes content") - assert extracted.get("content") == b"raw bytes content" - - def test_accepts_tuple(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_file_data, - ) - - extracted = extract_file_data(("foo.txt", b"raw bytes content")) - assert extracted.get("filename") == "foo.txt" - assert extracted.get("content") == b"raw bytes content" - - class TestUnpackLegacyDefs: """Cover the public ``unpack_legacy_defs`` helper directly so the no-op branches (non-dict input, schema with no legacy/OpenAPI defs) are exercised @@ -873,21 +392,25 @@ def test_hoist_images_from_tool_messages_bare_data_uri_string_passes_through(): [ ( {"type": "image_url", "image_url": {"url": DATA_URI_PNG}}, - TOOL_RESULT_IMAGE_PLACEHOLDER, TOOL_RESULT_IMAGE_BOUNDARY, + TOOL_RESULT_IMAGE_PLACEHOLDER, + TOOL_RESULT_IMAGE_BOUNDARY, ), ( {"type": "file", "file": {"file_data": "data:application/pdf;base64,cGRm", "filename": "report.pdf"}}, - TOOL_RESULT_MEDIA_PLACEHOLDER, TOOL_RESULT_MEDIA_BOUNDARY, + TOOL_RESULT_MEDIA_PLACEHOLDER, + TOOL_RESULT_MEDIA_BOUNDARY, ), ( {"type": "input_audio", "input_audio": {"data": "c291bmQ=", "format": "wav"}}, - TOOL_RESULT_MEDIA_PLACEHOLDER, TOOL_RESULT_MEDIA_BOUNDARY, + TOOL_RESULT_MEDIA_PLACEHOLDER, + TOOL_RESULT_MEDIA_BOUNDARY, ), ], ) def test_hoist_tool_media_preserves_payload_and_call( part: ChatCompletionImageObject | ChatCompletionFileObject | ChatCompletionAudioObject, - placeholder: str, boundary: str, + placeholder: str, + boundary: str, ) -> None: messages: Final = [ _assistant_tool_call_msg("call_1"), @@ -898,7 +421,8 @@ def test_hoist_tool_media_preserves_payload_and_call( result: Final = hoist_images_from_tool_messages(messages) assert result == [ - messages[0], {**messages[1], "content": placeholder}, + messages[0], + {**messages[1], "content": placeholder}, {"role": "user", "content": [{"type": "text", "text": boundary}, part]}, ] assert messages == original @@ -1062,104 +586,6 @@ class TestCustomToolFormatShapeConversion: # --- x-litellm-model upload-path decoding (litellm #29830) ------------------- -def _xlitellm_encoded(raw_id: str, model: str) -> str: - from litellm.proxy.openai_files_endpoints.common_utils import ( - encode_file_id_with_model, - ) - - return encode_file_id_with_model(raw_id, model) - - -def test_update_messages_with_model_file_ids_decodes_xlitellm_encoded_id(): - """x-litellm-model upload returns `file-;model,)>`. - Without decoding, the encoded id leaks to upstream OpenAI and errors as - 'Files [...] were not found'. Decode it back to raw provider id.""" - raw_id = "file-ExTuCawUqxEMjVFK6xwR9B" - encoded_id = _xlitellm_encoded(raw_id, "gpt-5.1") - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Summarize this."}, - {"type": "file", "file": {"file_id": encoded_id}}, - ], - } - ] - - updated = update_messages_with_model_file_ids(messages, "model-A", {}) - - assert updated[0]["content"][1]["file"]["file_id"] == raw_id - - -def test_update_responses_input_with_model_file_ids_decodes_xlitellm_encoded_id(): - """Same bug on /v1/responses path. Without decoding the encoded id (>64 - chars), OpenAI rejects with 'string too long. Expected ... maximum length - 64'.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - update_responses_input_with_model_file_ids, - ) - - raw_id = "file-ExTuCawUqxEMjVFK6xwR9B" - encoded_id = _xlitellm_encoded(raw_id, "gpt-5.1") - input_items = [ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "Summarize."}, - {"type": "input_file", "file_id": encoded_id}, - ], - } - ] - - updated = update_responses_input_with_model_file_ids(input_items) - - assert updated[0]["content"][1]["file_id"] == raw_id - - - -@pytest.mark.parametrize("unusable_file_id", [[1], {"a": 1}]) -def test_update_responses_input_with_model_file_ids_keeps_a_non_string_file_id_as_sent(unusable_file_id): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - update_responses_input_with_model_file_ids, - ) - - encoded_id = _xlitellm_encoded("file-ExTuCawUqxEMjVFK6xwR9B", "gpt-5.1") - input_items = [ - { - "role": "user", - "content": [ - {"type": "input_file", "file_id": encoded_id}, - {"type": "input_file", "file_id": unusable_file_id}, - ], - } - ] - - updated = update_responses_input_with_model_file_ids( - input_items, "model-A", {encoded_id: {"model-A": "file-on-a"}} - ) - - assert [item["file_id"] for item in updated[0]["content"]] == ["file-on-a", unusable_file_id] - -def test_update_messages_xlitellm_decode_does_not_override_mapping(): - """If the call-site already resolved a provider id via the mapping, that - wins. The new decode fallback runs only when no mapping match.""" - raw_id = "file-ExTuCawUqxEMjVFK6xwR9B" - encoded_id = _xlitellm_encoded(raw_id, "gpt-5.1") - mapping = {encoded_id: {"model-A": "provider-explicit-id"}} - messages = [ - { - "role": "user", - "content": [ - {"type": "file", "file": {"file_id": encoded_id}}, - ], - } - ] - - updated = update_messages_with_model_file_ids(messages, "model-A", mapping) - - assert updated[0]["content"][0]["file"]["file_id"] == "provider-explicit-id" - - def test_drop_tool_reference_parts_keeps_text_parts(): from litellm.litellm_core_utils.prompt_templates.common_utils import ( drop_tool_reference_parts_from_tool_messages, @@ -1259,719 +685,6 @@ class TestSystemMessagesFirst: assert system_messages_first(messages) == messages -class TestFlattenTopLevelSchemaCombinators: - def _customer_anyof_schema(self): - return { - "type": "object", - "anyOf": [ - { - "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, - "required": ["id", "enabled"], - }, - { - "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, - "required": ["id", "schedule"], - }, - ], - "properties": {"id": {"type": "string"}}, - "required": ["id"], - } - - def test_merges_anyof_branches_into_object_schema(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - result = flatten_top_level_schema_combinators(self._customer_anyof_schema()) - - assert "anyOf" not in result - assert result["type"] == "object" - assert set(result["properties"]) == {"id", "enabled", "schedule"} - assert result["properties"]["enabled"] == {"type": "boolean"} - assert result["required"] == ["id"] - - def test_typeless_anyof_of_object_branches_gets_intersected_required(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "anyOf": [ - { - "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, - "required": ["id", "enabled"], - }, - { - "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, - "required": ["id", "schedule"], - }, - ] - } - - result = flatten_top_level_schema_combinators(schema) - - assert result["type"] == "object" - assert "anyOf" not in result - assert result["required"] == ["id"] - - def test_allof_required_is_the_union_of_branches(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "type": "object", - "allOf": [ - {"properties": {"id": {"type": "string"}}, "required": ["id"]}, - {"properties": {"enabled": {"type": "boolean"}}, "required": ["enabled"]}, - ], - } - - result = flatten_top_level_schema_combinators(schema) - - assert "allOf" not in result - assert result["required"] == ["enabled", "id"] - assert set(result["properties"]) == {"id", "enabled"} - - def test_top_level_schema_wins_property_collisions(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "type": "object", - "anyOf": [ - {"properties": {"id": {"type": "integer"}}}, - {"properties": {"id": {"type": "number"}}}, - ], - "properties": {"id": {"type": "string"}}, - } - - result = flatten_top_level_schema_combinators(schema) - - assert result["properties"]["id"] == {"type": "string"} - - def test_drops_openai_rejected_scalar_keys_on_object_schema(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "type": "object", - "properties": {"id": {"type": "string"}}, - "enum": [{"id": "a"}], - "const": {"id": "a"}, - "not": {"required": ["other"]}, - } - - result = flatten_top_level_schema_combinators(schema) - - assert "enum" not in result - assert "const" not in result - assert "not" not in result - assert result["properties"] == {"id": {"type": "string"}} - - def test_resolves_local_ref_branches_from_defs(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "anyOf": [{"$ref": "#/$defs/Enable"}, {"$ref": "#/$defs/Schedule"}], - "$defs": { - "Enable": { - "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, - "required": ["id", "enabled"], - }, - "Schedule": { - "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, - "required": ["id", "schedule"], - }, - }, - } - - result = flatten_top_level_schema_combinators(schema) - - assert "anyOf" not in result - assert result["type"] == "object" - assert set(result["properties"]) == {"id", "enabled", "schedule"} - assert result["required"] == ["id"] - assert "$defs" in result - - def test_flattens_nested_combinator_branch_from_definitions(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "type": "object", - "oneOf": [ - {"$ref": "#/definitions/Toggle"}, - {"allOf": [{"properties": {"schedule": {"type": "string"}}, "required": ["schedule"]}]}, - ], - "definitions": {"Toggle": {"properties": {"enabled": {"type": "boolean"}}, "required": ["enabled"]}}, - } - - result = flatten_top_level_schema_combinators(schema) - - assert "oneOf" not in result - assert set(result["properties"]) == {"enabled", "schedule"} - assert "required" not in result - - def test_unresolvable_ref_branch_leaves_schema_untouched(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "type": "object", - "anyOf": [{"$ref": "https://example.com/schemas/automation.json"}], - "properties": {"id": {"type": "string"}}, - } - - assert flatten_top_level_schema_combinators(schema) is schema - - def test_self_referencing_ref_branch_leaves_schema_untouched(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "type": "object", - "anyOf": [{"$ref": "#/$defs/Node"}], - "$defs": {"Node": {"type": "object", "anyOf": [{"$ref": "#/$defs/Node"}]}}, - } - - assert flatten_top_level_schema_combinators(schema) is schema - - @pytest.mark.parametrize("boolean_branch", [True, False]) - def test_boolean_branch_leaves_schema_untouched(self, boolean_branch): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "type": "object", - "anyOf": [boolean_branch, {"properties": {"id": {"type": "string"}}, "required": ["id"]}], - } - - assert flatten_top_level_schema_combinators(schema) is schema - - def test_root_required_is_combined_with_branch_required(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - allof_schema = { - "type": "object", - "required": ["id"], - "properties": {"id": {"type": "string"}}, - "allOf": [{"properties": {"enabled": {"type": "boolean"}}, "required": ["enabled"]}], - } - anyof_schema = { - "type": "object", - "required": ["id"], - "properties": {"id": {"type": "string"}}, - "anyOf": [ - {"properties": {"name": {"type": "string"}, "a": {"type": "string"}}, "required": ["name", "a"]}, - {"properties": {"name": {"type": "string"}, "b": {"type": "string"}}, "required": ["name", "b"]}, - ], - } - - assert flatten_top_level_schema_combinators(allof_schema)["required"] == ["enabled", "id"] - assert flatten_top_level_schema_combinators(anyof_schema)["required"] == ["id", "name"] - - def test_repeated_refs_are_expanded_once(self): - import time - - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - fan_out, chain_length = 8, 8 - schema = { - "type": "object", - "anyOf": [{"$ref": "#/$defs/Level0"}], - "$defs": { - **{ - f"Level{level}": {"anyOf": [{"$ref": f"#/$defs/Level{level + 1}"}] * fan_out} - for level in range(chain_length) - }, - f"Level{chain_length}": {"type": "object", "properties": {"id": {"type": "string"}}}, - }, - } - - started = time.perf_counter() - result = flatten_top_level_schema_combinators(schema) - - assert time.perf_counter() - started < 5 - assert "anyOf" not in result - assert result["properties"] == {"id": {"type": "string"}} - - def test_nesting_past_the_depth_cap_leaves_schema_untouched(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - def nested(levels): - leaf = {"type": "object", "properties": {"id": {"type": "string"}}} - return functools.reduce(lambda inner, _: {"type": "object", "anyOf": [inner]}, range(levels), leaf) - - shallow, deep = nested(20), nested(40) - - assert "anyOf" not in flatten_top_level_schema_combinators(shallow) - assert flatten_top_level_schema_combinators(deep) is deep - - @pytest.mark.parametrize( - "branches", - [ - [{"required": ["enabled"]}, {"required": ["schedule"]}], - [{"type": "object", "required": ["enabled"]}, {"type": "object", "required": ["schedule"]}], - ], - ) - def test_typeless_root_with_properties_flattens_branches_without_properties(self, branches): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}, "schedule": {"type": "string"}}, - "required": ["id"], - "anyOf": branches, - } - - result = flatten_top_level_schema_combinators(schema) - - assert "anyOf" not in result - assert result["type"] == "object" - assert set(result["properties"]) == {"id", "enabled", "schedule"} - assert result["required"] == ["id"] - - def test_typeless_root_flattens_typed_object_branches_without_properties(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = { - "anyOf": [ - {"type": "object", "properties": {"id": {"type": "string"}}}, - {"type": "object", "required": ["id"]}, - ] - } - - result = flatten_top_level_schema_combinators(schema) - - assert "anyOf" not in result - assert result["type"] == "object" - assert result["properties"] == {"id": {"type": "string"}} - assert "required" not in result - - def test_non_object_union_passes_through_unchanged(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = {"anyOf": [{"type": "string"}, {"type": "number"}]} - - assert flatten_top_level_schema_combinators(schema) is schema - - def test_schema_without_rejected_keys_is_returned_as_is(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = {"type": "object", "properties": {"nested": {"anyOf": [{"type": "string"}, {"type": "null"}]}}} - - assert flatten_top_level_schema_combinators(schema) is schema - - def test_input_schema_is_never_mutated(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_top_level_schema_combinators, - ) - - schema = self._customer_anyof_schema() - snapshot = json.loads(json.dumps(schema)) - - flatten_top_level_schema_combinators(schema) - - assert schema == snapshot - - -class TestToolWithSanitizedParameters: - def _anyof_tool(self): - return { - "type": "function", - "function": { - "name": "automation_update", - "description": "Update an automation", - "parameters": { - "type": "object", - "anyOf": [ - { - "properties": {"id": {"type": "string"}, "enabled": {"type": "boolean"}}, - "required": ["id", "enabled"], - }, - { - "properties": {"id": {"type": "string"}, "schedule": {"type": "string"}}, - "required": ["id", "schedule"], - }, - ], - "properties": {"id": {"type": "string"}}, - "required": ["id"], - }, - }, - } - - def test_flattens_anyof_parameters_into_new_tool(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_combinators_and_drop_non_python_regex_patterns, - tool_with_sanitized_parameters, - ) - - tool = self._anyof_tool() - result = tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) - - assert result is not tool - parameters = result["function"]["parameters"] - assert "anyOf" not in parameters - assert parameters["type"] == "object" - assert set(parameters["properties"]) == {"id", "enabled", "schedule"} - assert parameters["required"] == ["id"] - assert result["function"]["name"] == "automation_update" - assert tool == self._anyof_tool() - - def test_clean_parameters_return_the_same_tool_object(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_combinators_and_drop_non_python_regex_patterns, - tool_with_sanitized_parameters, - ) - - tool = { - "type": "function", - "function": { - "name": "lookup", - "parameters": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}, - }, - } - - assert tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) is tool - - def test_pattern_only_sanitizer_drops_the_regex_and_keeps_the_union(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_non_python_regex_patterns, - tool_with_sanitized_parameters, - ) - - tool = self._anyof_tool() - tool["function"]["parameters"]["properties"]["id"]["pattern"] = _ARTIFACT_FIELD_PATTERN - - result = tool_with_sanitized_parameters(tool, drop_non_python_regex_patterns) - - parameters = result["function"]["parameters"] - assert parameters["properties"]["id"] == {"type": "string"} - assert parameters["anyOf"] == self._anyof_tool()["function"]["parameters"]["anyOf"] - assert tool["function"]["parameters"]["properties"]["id"]["pattern"] == _ARTIFACT_FIELD_PATTERN - - @pytest.mark.parametrize( - "tool", - [ - {"type": "function"}, - {"type": "function", "function": "not-a-dict"}, - {"type": "function", "function": {"name": "no_params"}}, - {"type": "function", "function": {"name": "bad_params", "parameters": "not-a-dict"}}, - ], - ) - def test_non_dict_function_or_parameters_return_the_same_tool_object(self, tool): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - flatten_combinators_and_drop_non_python_regex_patterns, - tool_with_sanitized_parameters, - ) - - assert tool_with_sanitized_parameters(tool, flatten_combinators_and_drop_non_python_regex_patterns) is tool - - def test_sanitizes_the_input_schema_of_an_anthropic_tool(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_lookaround_regex_patterns, - tool_with_sanitized_parameters, - ) - - tool = { - "name": "ArtifactData", - "description": "Read a shared database", - "input_schema": { - "type": "object", - "properties": {"doc_id": {"type": "string", "pattern": _ARTIFACT_DATA_ID_PATTERN}}, - }, - } - - result = tool_with_sanitized_parameters(tool, drop_lookaround_regex_patterns) - - assert result == { - "name": "ArtifactData", - "description": "Read a shared database", - "input_schema": {"type": "object", "properties": {"doc_id": {"type": "string"}}}, - } - assert tool["input_schema"]["properties"]["doc_id"]["pattern"] == _ARTIFACT_DATA_ID_PATTERN - - def test_returns_the_same_anthropic_tool_when_its_schema_has_nothing_to_drop(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_lookaround_regex_patterns, - tool_with_sanitized_parameters, - ) - - tool = {"name": "Read", "input_schema": {"type": "object", "properties": {"path": {"type": "string"}}}} - - assert tool_with_sanitized_parameters(tool, drop_lookaround_regex_patterns) is tool - - -def _regex_schema(pattern): - return { - "type": "object", - "properties": { - "field": {"type": "string", "pattern": pattern}, - "writes": { - "type": "array", - "items": {"properties": {"doc_id": {"type": "string", "pattern": pattern}}}, - }, - "query": {"anyOf": [{"type": "string", "pattern": pattern}, {"type": "null"}]}, - "pair": {"type": "array", "prefixItems": [{"type": "string", "pattern": pattern}]}, - "extra": {"type": "object", "additionalProperties": {"type": "string", "pattern": pattern}}, - "tagged": { - "type": "object", - "patternProperties": {pattern: {"type": "string"}, "^x_": {"type": "integer"}}, - }, - }, - "$defs": {"segment": {"type": "string", "pattern": pattern}}, - "required": ["field"], - } - - -class TestDropNonPythonRegexPatterns: - """Claude Code's Artifact tool declares ECMA-262 ``\\p{..}`` escapes that OpenAI's - validator, which compiles ``pattern`` values and ``patternProperties`` keys with - Python ``re``, refuses as "not a 'regex'".""" - - def test_drops_every_regex_python_re_rejects_from_every_schema_position(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_non_python_regex_patterns, - ) - - schema = _regex_schema(_ARTIFACT_FIELD_PATTERN) - - result = drop_non_python_regex_patterns(schema) - - assert '"pattern"' not in json.dumps(result) - properties = result["properties"] - assert properties["field"] == {"type": "string"} - assert properties["writes"]["items"]["properties"]["doc_id"] == {"type": "string"} - assert properties["query"]["anyOf"] == [{"type": "string"}, {"type": "null"}] - assert properties["pair"]["prefixItems"] == [{"type": "string"}] - assert properties["extra"]["additionalProperties"] == {"type": "string"} - assert properties["tagged"]["patternProperties"] == {"^x_": {"type": "integer"}} - assert result["$defs"]["segment"] == {"type": "string"} - assert result["required"] == ["field"] - assert schema == _regex_schema(_ARTIFACT_FIELD_PATTERN) - - def test_keeps_regexes_python_re_compiles_and_returns_the_same_object(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_non_python_regex_patterns, - ) - - schema = _regex_schema(r'^(?!__.*__$)[^"\\./[\]]{1,200}$') - - assert drop_non_python_regex_patterns(schema) is schema - - def test_pattern_keys_inside_data_positions_are_not_regexes(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_non_python_regex_patterns, - ) - - schema = { - "type": "object", - "properties": { - "pattern": {"type": "string"}, - "template": {"type": "object", "default": {"pattern": _ARTIFACT_FIELD_PATTERN}}, - "samples": {"type": "array", "examples": [{"pattern": _ARTIFACT_FIELD_PATTERN}]}, - "fixed": {"const": {"pattern": _ARTIFACT_FIELD_PATTERN}}, - "vendor": {"type": "string", "x-litellm": {"pattern": _ARTIFACT_FIELD_PATTERN}}, - }, - "required": ["pattern"], - } - - assert drop_non_python_regex_patterns(schema) is schema - - def test_regex_nested_past_what_python_re_can_parse_is_dropped_not_raised(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_non_python_regex_patterns, - ) - - schema = { - "type": "object", - "properties": {"deep": {"type": "string", "pattern": "(" * 2000 + "a" + ")" * 2000}}, - } - - assert drop_non_python_regex_patterns(schema)["properties"]["deep"] == {"type": "string"} - - def test_walks_schemas_deeper_than_the_interpreter_recursion_limit(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_non_python_regex_patterns, - ) - - depth = sys.getrecursionlimit() - leaf = {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN} - schema = functools.reduce( - lambda inner, _: {"type": "object", "properties": {"child": inner}}, range(depth), leaf - ) - - result = drop_non_python_regex_patterns(schema) - - assert functools.reduce(lambda node, _: node["properties"]["child"], range(depth), result) == {"type": "string"} - assert functools.reduce(lambda node, _: node["properties"]["child"], range(depth), schema) is leaf - - def test_leaves_levels_past_the_json_nesting_limit_alone(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_non_python_regex_patterns, - ) - - leaf = {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN} - schema = functools.reduce( - lambda inner, _: {"type": "object", "properties": {"child": inner}}, range(1100), leaf - ) - - assert drop_non_python_regex_patterns(schema) is schema - - -class TestDropLookaroundRegexPatterns: - """Kimi K3 and Grok 4.6/4.7 on Bedrock Converse reject every tool schema regex that - uses a lookaround assertion, Claude Code's ``ArtifactData`` ``pattern`` included.""" - - @pytest.mark.parametrize( - "pattern", - [r"^(?!x).*$", r"^(?=.*a).*$", r"^.*(?\w+)$", r"^(?i)abc$", r"^[^\p{Cc}\p{Cf}]{1,200}$"], - ids=["plain", "non-capturing-group", "named-group", "inline-flag", "non-python-without-lookaround"], - ) - def test_keeps_regexes_without_lookaround_and_returns_the_same_object(self, pattern): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_lookaround_regex_patterns, - ) - - schema = _regex_schema(pattern) - - assert drop_lookaround_regex_patterns(schema) is schema - - def test_lookaround_inside_data_positions_is_not_a_regex(self): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_lookaround_regex_patterns, - ) - - schema = { - "type": "object", - "properties": { - "pattern": {"type": "string"}, - "template": {"type": "object", "default": {"pattern": _ARTIFACT_DATA_ID_PATTERN}}, - "hint": {"type": "string", "description": "ids match " + _ARTIFACT_DATA_ID_PATTERN}, - }, - "required": ["pattern"], - } - - assert drop_lookaround_regex_patterns(schema) is schema - - -@pytest.mark.parametrize( - ("dropper", "patterns"), - [ - ("drop_non_python_regex_patterns", (_ARTIFACT_FIELD_PATTERN, r"^\p{L}+$")), - ("drop_lookaround_regex_patterns", (_ARTIFACT_DATA_ID_PATTERN, r"^(?=.*[a-z])\w+$")), - ], - ids=["non-python", "lookaround"], -) -class TestDroppedPatternPropertiesKeepTheirNamesAllowed: - """Dropping a ``patternProperties`` key from an object closed by ``additionalProperties: - false`` must not ban the names that key allowed: its value schema takes over as the - object's ``additionalProperties``.""" - - @staticmethod - def _drop(dropper): - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - drop_lookaround_regex_patterns, - drop_non_python_regex_patterns, - ) - - return { - "drop_non_python_regex_patterns": drop_non_python_regex_patterns, - "drop_lookaround_regex_patterns": drop_lookaround_regex_patterns, - }[dropper] - - def test_closed_object_takes_the_dropped_value_schema(self, dropper, patterns): - schema = { - "type": "object", - "patternProperties": {patterns[0]: {"type": "string", "pattern": patterns[0]}}, - "additionalProperties": False, - } - - assert self._drop(dropper)(schema) == { - "type": "object", - "patternProperties": {}, - "additionalProperties": {"type": "string"}, - } - - def test_closed_object_losing_two_entries_accepts_either_value_schema(self, dropper, patterns): - schema = { - "type": "object", - "patternProperties": { - patterns[0]: {"type": "string"}, - patterns[1]: {"type": "integer"}, - "^x_": {"type": "boolean"}, - }, - "additionalProperties": False, - } - - assert self._drop(dropper)(schema) == { - "type": "object", - "patternProperties": {"^x_": {"type": "boolean"}}, - "additionalProperties": {"anyOf": [{"type": "string"}, {"type": "integer"}]}, - } - - def test_object_with_its_own_additional_properties_schema_keeps_it(self, dropper, patterns): - schema = { - "type": "object", - "patternProperties": {patterns[0]: {"type": "string"}}, - "additionalProperties": {"type": "integer"}, - } - - assert self._drop(dropper)(schema) == { - "type": "object", - "patternProperties": {}, - "additionalProperties": {"type": "integer"}, - } - - class TestRequestContainsImageContent: """One detector for every dialect that reaches pre-routing hooks untranslated.""" @@ -2025,134 +738,6 @@ class TestRequestContainsImageContent: assert request_contains_image_content([{"role": "user", "content": [nested]}]) is False -class TestEncryptedReasoningReplay: - """Regression for https://github.com/BerriAI/litellm/issues/40288.""" - - def test_signature_round_trips_the_encrypted_content(self): - assert encrypted_content_from_signature(encrypted_reasoning_signature("gAAAA_bytes")) == "gAAAA_bytes" - - @pytest.mark.parametrize( - "signature", [None, "", "ErcBCkgIValidAnthropicSignature", "litellm_encrypted_reasoning:", 7] - ) - def test_anything_else_is_not_encrypted_content(self, signature): - assert encrypted_content_from_signature(signature) is None - - def test_encrypted_thinking_block_replays_its_own_item(self): - items = responses_reasoning_items_from_thinking_blocks( - [{"type": "thinking", "thinking": "Plan.", "signature": encrypted_reasoning_signature("gAAAA_1")}] - ) - assert items == ( - { - "type": "reasoning", - "summary": [{"type": "summary_text", "text": "Plan."}], - "encrypted_content": "gAAAA_1", - }, - ) - - def test_encrypted_redacted_block_replays_with_an_empty_summary(self): - items = responses_reasoning_items_from_thinking_blocks( - [{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_1")}] - ) - assert items == ({"type": "reasoning", "summary": [], "encrypted_content": "gAAAA_1"},) - - def test_plain_blocks_collapse_into_one_summary_item_around_encrypted_ones(self): - items = responses_reasoning_items_from_thinking_blocks( - [ - {"type": "thinking", "thinking": "A.", "signature": None}, - {"type": "thinking", "thinking": "B.", "signature": ""}, - {"type": "thinking", "thinking": "C.", "signature": encrypted_reasoning_signature("gAAAA_c")}, - {"type": "redacted_thinking", "data": "anthropic-minted-opaque-data"}, - {"type": "thinking", "thinking": "D."}, - ] - ) - assert items == ( - { - "type": "reasoning", - "summary": [{"type": "summary_text", "text": "A."}, {"type": "summary_text", "text": "B."}], - }, - {"type": "reasoning", "summary": [{"type": "summary_text", "text": "C."}], "encrypted_content": "gAAAA_c"}, - {"type": "reasoning", "summary": [{"type": "summary_text", "text": "D."}]}, - ) - assert all("id" not in item for item in items) - - def test_blocks_without_text_or_encrypted_content_produce_nothing(self): - assert responses_reasoning_items_from_thinking_blocks([{"type": "thinking", "thinking": ""}]) == () - assert responses_reasoning_items_from_thinking_blocks([]) == () - - @pytest.mark.parametrize( - ("block", "expected"), - [ - ({"type": "thinking", "thinking": "x", "signature": encrypted_reasoning_signature("g")}, True), - ({"type": "redacted_thinking", "data": encrypted_reasoning_signature("g")}, True), - ({"type": "thinking", "thinking": "x", "signature": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True), - ({"type": "redacted_thinking", "data": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True), - ({"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}, False), - ({"type": "redacted_thinking", "data": "EmwKAhgBEgy"}, False), - ({"type": "text", "text": encrypted_reasoning_signature("g")}, False), - ("not a block", False), - ], - ) - def test_is_encrypted_reasoning_block(self, block, expected): - assert is_encrypted_reasoning_block(block) is expected - - def test_strip_drops_every_bridge_tagged_block_and_leaves_no_unsigned_thinking_behind(self): - assistant_content = [ - {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, - {"type": "thinking", "thinking": "packed by the bridge", "signature": encrypted_reasoning_signature("g1")}, - {"type": "redacted_thinking", "data": encrypted_reasoning_signature("g2")}, - {"type": "thinking", "thinking": "", "signature": encrypted_reasoning_signature("g3")}, - {"type": "text", "text": "answer"}, - ] - messages = [ - {"role": "user", "content": "question"}, - {"role": "assistant", "content": assistant_content}, - {"role": "user", "content": [{"type": "text", "text": "follow-up"}]}, - ] - - strip_encrypted_reasoning_from_messages(messages) - - assert messages[1]["content"] is assistant_content - assert assistant_content == [ - {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, - {"type": "text", "text": "answer"}, - ] - assert all(block["signature"] for block in assistant_content if block["type"] == "thinking") - assert messages[0] == {"role": "user", "content": "question"} - assert messages[2] == {"role": "user", "content": [{"type": "text", "text": "follow-up"}]} - - def test_strip_uses_predicate_to_keep_selected_encrypted_blocks(self): - kept_signature = encrypted_reasoning_signature("keep") - stripped_signature = encrypted_reasoning_signature("strip") - content = [ - {"type": "thinking", "thinking": "keep", "signature": kept_signature}, - {"type": "thinking", "thinking": "strip", "signature": stripped_signature}, - ] - messages = [{"role": "assistant", "content": content}] - - strip_encrypted_reasoning_from_messages(messages, should_strip=lambda block: block.get("thinking") == "strip") - - assert messages[0]["content"] is content - assert content == [{"type": "thinking", "thinking": "keep", "signature": kept_signature}] - - @pytest.mark.parametrize( - "messages", - [ - "not a list", - None, - [{"role": "user", "content": None}], - [{"role": "user", "content": "plain string"}], - ["not a message"], - [{"role": "assistant", "content": [{"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}]}], - ], - ) - def test_strip_leaves_history_without_bridge_reasoning_untouched(self, messages): - before = copy.deepcopy(messages) - - strip_encrypted_reasoning_from_messages(messages) - - assert messages == before - - class TestMergeConsecutiveSystemMessages: def test_merges_each_run_of_string_system_messages_with_a_blank_line(self): messages = [ @@ -2337,132 +922,6 @@ def test_get_semantic_cache_prompt_from_messages_matches_get_str_from_messages_w assert get_semantic_cache_prompt_from_messages(messages) == get_str_from_messages(messages) -_MALFORMED_PROMPT_CACHE_BREAKPOINTS: Final = ( - "yes", - 1, - ["explicit"], - "", - {}, - {"mode": "bogus"}, - {"mode": "bogus", "ttl": "30m"}, - {1: "explicit"}, -) -_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS: Final = ( - "string", - "int", - "list", - "empty-string", - "empty-object", - "bogus-mode", - "bogus-mode-with-ttl", - "non-string-key", -) -_VALID_PROMPT_CACHE_BREAKPOINTS: Final = ({"mode": "explicit"}, {"mode": "explicit", "ttl": "30m"}) - - -@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep")) -def test_prompt_cache_breakpoint_for_wire_passes_a_missing_marker_through(drop_params: bool) -> None: - assert prompt_cache_breakpoint_for_wire(None, drop_params) is None - - -@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS) -def test_prompt_cache_breakpoint_for_wire_drops_a_malformed_marker_under_drop_params(marker: object) -> None: - assert prompt_cache_breakpoint_for_wire(marker, True) is None - - -@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS) -def test_prompt_cache_breakpoint_for_wire_forwards_a_malformed_marker_without_drop_params(marker: object) -> None: - assert prompt_cache_breakpoint_for_wire(marker, False) is marker - - -@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep")) -@pytest.mark.parametrize("marker", _VALID_PROMPT_CACHE_BREAKPOINTS, ids=("explicit", "explicit-30m")) -def test_prompt_cache_breakpoint_for_wire_keeps_a_valid_marker(marker: dict[str, str], drop_params: bool) -> None: - assert prompt_cache_breakpoint_for_wire(marker, drop_params) == marker - - -def test_prompt_cache_breakpoint_for_wire_strips_an_unknown_key_under_drop_params() -> None: - assert prompt_cache_breakpoint_for_wire({"mode": "explicit", "note": "kept"}, True) == {"mode": "explicit"} - - -@pytest.mark.parametrize( - "marker", - ({"mode": "explicit", "ttl": "1h"}, {"mode": "explicit", "ttl": "1h", "note": "kept"}), - ids=("bad-ttl", "bad-ttl-and-unknown-key"), -) -def test_prompt_cache_breakpoint_for_wire_drops_an_unsupported_ttl_and_keeps_the_mode_under_drop_params( - marker: dict[str, str], -) -> None: - assert prompt_cache_breakpoint_for_wire(marker, True) == {"mode": "explicit"} - assert prompt_cache_breakpoint_for_wire(marker, False) is marker - - -def test_cache_control_is_replaced_by_an_explicit_prompt_cache_breakpoint_on_the_same_block() -> None: - marked: Final = {"type": "text", "text": "rules", "cache_control": {"type": "ephemeral", "ttl": "1h"}} - - assert cache_control_as_prompt_cache_breakpoint(marked) == { - "type": "text", - "text": "rules", - "prompt_cache_breakpoint": {"mode": "explicit"}, - } - - -def test_a_block_without_cache_control_is_returned_untouched() -> None: - unmarked: Final = {"type": "text", "text": "rules"} - - assert cache_control_as_prompt_cache_breakpoint(unmarked) is unmarked - - -def test_a_breakpoint_the_client_wrote_survives_a_message_level_mark_ttl_included() -> None: - block: Final = { - "type": "input_text", - "text": "rules", - "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}, - } - - assert with_prompt_cache_breakpoint(block, {"mode": "explicit"}) is block - - -def test_a_breakpoint_the_client_wrote_survives_a_per_part_cache_control() -> None: - both: Final = { - "type": "input_text", - "text": "rules", - "cache_control": {"type": "ephemeral"}, - "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}, - } - - assert cache_control_as_prompt_cache_breakpoint(both) == { - "type": "input_text", - "text": "rules", - "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}, - } - - -_NATIVE_REDACTED_DATA: Final = "EmwKAhgBEgy3va3pzix/LafPsn4aDFIT2Xlxh0L5L8rLVyIwxtE3rAFBa8cr3qpP" -_CONVERSE_REDACTED_DATA: Final = base64.b64encode(_NATIVE_REDACTED_DATA.encode()).decode() - - -@pytest.mark.parametrize( - "data, native", - [ - pytest.param(_NATIVE_REDACTED_DATA, _NATIVE_REDACTED_DATA, id="claude-native"), - pytest.param(_CONVERSE_REDACTED_DATA, _NATIVE_REDACTED_DATA, id="converse-wrapped-legacy-history"), - pytest.param( - encrypted_reasoning_signature("gAAAA_1"), encrypted_reasoning_signature("gAAAA_1"), id="openai-encrypted" - ), - pytest.param("", "", id="blank"), - pytest.param(base64.b64encode(b"not base64!").decode(), base64.b64encode(b"not base64!").decode(), id="b64-once"), - ], -) -def test_native_redacted_thinking_data_only_unwraps_the_converse_base64_layer(data: str, native: str) -> None: - assert native_redacted_thinking_data(data) == native - - -@pytest.mark.parametrize("data", [_NATIVE_REDACTED_DATA, _CONVERSE_REDACTED_DATA], ids=["native", "legacy-wrapped"]) -def test_converse_redacted_content_wraps_claude_native_data_exactly_once(data: str) -> None: - assert converse_redacted_content(data) == _CONVERSE_REDACTED_DATA - - def test_strip_litellm_response_only_fields_drops_nested_tool_call_provider_fields() -> None: message: Final = { "role": "assistant", @@ -2490,78 +949,6 @@ def test_strip_litellm_response_only_fields_drops_nested_tool_call_provider_fiel assert function["arguments"] == '{"city": "Paris"}' -@pytest.mark.parametrize( - ("schema", "is_open"), - [ - ({"type": "object", "properties": {"a": {"type": "string"}}}, False), - ({"type": "object", "properties": {}}, False), - ({"type": "object", "additionalProperties": False}, False), - ({"type": "string"}, False), - ({"type": "object"}, True), - ({"type": ["object", "null"]}, True), - ({"type": "object", "properties": {"a": {"type": "string"}}, "additionalProperties": True}, True), - ({"type": "object", "additionalProperties": {"type": "integer"}}, True), - ({"type": "object", "properties": {}, "patternProperties": {"^x": {"type": "string"}}}, True), - ({"type": "object", "properties": {"inner": {"anyOf": [{"type": "object"}, {"type": "null"}]}}}, True), - ({"type": "array", "items": {"type": "object", "additionalProperties": {"type": "string"}}}, True), - ], -) -def test_schema_has_open_object_flags_any_node_that_accepts_unlisted_keys(schema, is_open): - assert schema_has_open_object(schema) is is_open - - -def _object_with(properties: Mapping[str, object], **defs: Mapping[str, object]) -> Mapping[str, object]: - return {"type": "object", "properties": properties, **defs} - - -@pytest.mark.parametrize( - ("schema", "unresolvable"), - [ - (_object_with({"a": {"type": "string"}}), False), - (_object_with({"a": {"$ref": "#/$defs/A"}, "b": {"$ref": "#/$defs/A"}}, **{"$defs": {"A": {"type": "string"}}}), False), - (_object_with({"a": {"$ref": "#/definitions/A"}}, definitions={"A": {"$ref": "#/definitions/B"}, "B": {"type": "integer"}}), False), - (_object_with({"a": {"$ref": "/$defs/A"}}, **{"$defs": {"A": {"items": {"$ref": "/$defs/B"}}, "B": {"type": "string"}}}), False), - (_object_with({"a": {"$ref": "#/$defs/Missing"}}, **{"$defs": {"A": {"type": "string"}}}), True), - (_object_with({"a": {"$ref": "other.json#/A"}}), True), - (_object_with({"parent": {"$ref": "#"}}), True), - (_object_with({"a": {"$ref": "#/$defs/A"}}, **{"$defs": {"A": {"type": "array", "items": {"$ref": "#/$defs/A"}}}}), True), - (_object_with({"a": {"$ref": "/$defs/A"}}, **{"$defs": {"A": {"type": "array", "items": {"$ref": "/$defs/A"}}}}), True), - ( - _object_with( - {"a": {"$ref": "#/$defs/A"}}, - **{"$defs": {"A": {"properties": {"b": {"$ref": "#/$defs/B"}}}, "B": {"properties": {"a": {"$ref": "#/$defs/A"}}}}}, - ), - True, - ), - ], - ids=["no-refs", "shared-def", "chained-definitions", "root-path-defs", "missing-def", "external", "root", "self-cycle", "root-path-self-cycle", "mutual-cycle"], -) -def test_schema_has_unresolvable_ref_flags_refs_that_cannot_be_inlined(schema, unresolvable): - assert schema_has_unresolvable_ref(schema) is unresolvable - - -@pytest.mark.parametrize( - ("schema", "properties", "branches"), - [ - (_object_with({"a": {"type": "string"}, "b": {"type": "string"}, "c": {"type": "string"}}, required=["a"]), 3, 8), - ( - _object_with( - {"x": {"$ref": "#/$defs/Pair"}, "y": {"$ref": "#/$defs/Pair"}}, - required=["x", "y"], - **{"$defs": {"Pair": _object_with({"l": {"type": "string"}, "r": {"type": "string"}}, required=["l"])}}, - ), - 4, - 7, - ), - (_object_with({"inner": _object_with({"z": {"type": "string"}})}, required=["inner"]), 2, 4), - ], - ids=["flat", "shared-def-counted-once", "nested"], -) -def test_schema_size_counts_every_written_object_once(schema, properties, branches): - assert schema_property_count(schema) == properties - assert schema_grammar_branch_count(schema) == branches - - @pytest.mark.parametrize("part_type", ["image_url", "input_image"]) @pytest.mark.parametrize( "fields", @@ -2572,7 +959,8 @@ def test_schema_size_counts_every_written_object_once(schema, properties, branch ], ) def test_shared_image_url_agrees_with_responses_normalization( - part_type: str, fields: Mapping[str, object], + part_type: str, + fields: Mapping[str, object], ) -> None: from litellm.litellm_core_utils.prompt_templates.common_utils import as_openai_image_part from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig @@ -2584,121 +972,6 @@ def test_shared_image_url_agrees_with_responses_normalization( assert as_openai_image_part(normalized) == expected -@pytest.mark.parametrize( - "part,projected,media", - [ - ( - {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "high"}}, - {"type": "image_url", "image_url": {"url": "", "detail": "high"}}, - {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "high"}}, - ), - ( - {"type": "input_image", "image_url": "https://example.com/image.png", "detail": "low"}, - {"type": "input_image", "image_url": "", "detail": "low"}, - {"type": "image_url", "image_url": {"url": "https://example.com/image.png", "detail": "low"}}, - ), - ( - {"type": "computer_screenshot", "image_url": "data:image/png;base64,IMAGE"}, - {"type": "computer_screenshot", "image_url": ""}, - {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, - ), - ( - {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "IMAGE"}}, - {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}}, - {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, - ), - ( - {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}, - {"type": "input_audio", "input_audio": {"data": "", "format": "wav"}}, - {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}, - ), - ( - {"type": "file", "file": {"file_data": "IMAGE", "format": "image/png", "filename": "image.png"}}, - {"type": "file", "file": {"file_data": "", "format": "image/png", "filename": "image.png"}}, - {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, - ), - ( - {"type": "input_file", "file_data": "data:audio/wav;base64,AUDIO", "filename": "audio.wav"}, - {"type": "input_file", "file_data": "", "filename": "audio.wav"}, - {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}, - ), - ], -) -def test_tpm_projection_blanks_only_declared_media_transport( - part: Mapping[str, object], projected: Mapping[str, object], media: Mapping[str, object], -) -> None: - text: Final = {"type": "text", "text": "data:image/png;base64,IMAGE"} - request: Final = [{"role": "user", "content": [text, {**part, "cache_control": {"type": "ephemeral"}}]}] - original: Final = copy.deepcopy(request) - assert split_media_for_tpm(request) == ( - [{"role": "user", "content": [text, {**projected, "cache_control": {"type": "ephemeral"}}]}], - (media,), - ) - assert request == original - - -def test_tpm_projection_retains_nested_and_native_history_around_media() -> None: - image: Final = {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "high"}} - empty_image: Final = {"type": "image_url", "image_url": {"url": "", "detail": "high"}} - document: Final = {"type": "document", "title": "keep", "source": {"type": "content", "content": [image]}} - empty_document: Final = {**document, "source": {"type": "content", "content": [empty_image]}} - tool: Final = {"type": "tool_result", "tool_use_id": "call", "content": [document, {"type": "text", "text": "keep"}]} - empty_tool: Final = {**tool, "content": [empty_document, {"type": "text", "text": "keep"}]} - request: Final = [ - {"role": "user", "content": [tool]}, - {"type": "function_call_output", "call_id": "call", "output": [image]}, - {"type": "custom_tool_call_output", "call_id": "custom", "output": [image]}, - {"type": "computer_call_output", "call_id": "computer", "output": { - "type": "computer_screenshot", "image_url": "data:image/png;base64,IMAGE", - }}, - {"type": "image_generation_call", "id": "ig", "result": "IMAGE", "status": "completed"}, - {"type": "code_interpreter_call", "id": "ci", "code": "keep()", "outputs": [ - {"type": "logs", "logs": "keep"}, {"type": "image", "url": "https://example.com/image.png"}, - ]}, - ] - original: Final = copy.deepcopy(request) - projected, media = split_media_for_tpm(request) - assert projected == [ - {"role": "user", "content": [empty_tool]}, - {"type": "function_call_output", "call_id": "call", "output": [empty_image]}, - {"type": "custom_tool_call_output", "call_id": "custom", "output": [empty_image]}, - {"type": "computer_call_output", "call_id": "computer", "output": { - "type": "computer_screenshot", "image_url": "", - }}, - {"type": "image_generation_call", "id": "ig", "result": "", "status": "completed"}, - {"type": "code_interpreter_call", "id": "ci", "code": "keep()", "outputs": [ - {"type": "logs", "logs": "keep"}, {"type": "image", "url": ""}, - ]}, - ] - assert media == ( - image, image, image, - {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, - {"type": "image_url", "image_url": {"url": "data:application/octet-stream;base64,IMAGE", "detail": "auto"}}, - {"type": "image_url", "image_url": {"url": "https://example.com/image.png", "detail": "auto"}}, - ) - assert request == original - - -def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None: - image: Final = {"type": "input_image", "image_url": "data:image/png;base64,OPAQUE"} - request: Final = [ - {"role": "user", "content": json.dumps(image)}, - {"type": "custom_tool_call_output", "call_id": "call", "output": json.dumps(image)}, - {"type": "function_call", "call_id": "call", "name": "keep", "arguments": {"content": [image]}}, - {"type": "file_search_call", "results": [{"text": "keep", "content": [image]}]}, - {"type": "image_generation_call", "id": "ig", "status": "in_progress", "result": None}, - {"role": "user", "content": [ - {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "PDF"}}, - {"type": "document", "source": {"type": "text", "data": json.dumps(image)}}, - {"type": "file", "file": {"format": "application/pdf", "file_data": "PDF"}}, - {"type": "text", "text": "keep", "metadata": {"content": [image]}}, - ]}, - ] - projected, media = split_media_for_tpm(request) - assert projected is request - assert media == () - - def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_untouched() -> None: messages: Final = [ {"role": "user", "content": [{"type": "text", "text": "Hello"}, {"type": "text", "text": " world"}]}, @@ -2710,22 +983,3 @@ def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_unt assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}] assert messages == messages_before - - -@pytest.mark.parametrize("mime", ["application/pdf", "video/mp4", "audio/wav", "image/png"]) -def test_embedding_media_split_holds_at_most_one_copy_of_a_large_payload(mime: str) -> None: - import gc - import tracemalloc - - uri: Final = f"data:{mime};base64," + base64.b64encode(bytes(3 * 1024 * 1024)).decode("ascii") - split_embedding_media_for_tpm(f"data:{mime};base64,AAAA") - gc.collect() - tracemalloc.start() - try: - _, media = split_embedding_media_for_tpm(uri) - _, peak = tracemalloc.get_traced_memory() - finally: - tracemalloc.stop() - - assert media - assert peak < 1.5 * len(uri) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index d65f094971a..bb88f632073 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1,4 +1,6 @@ -import asyncio, base64, importlib, uuid +import asyncio +import base64 +import importlib import json import logging import os @@ -6,8 +8,10 @@ import re import sys import time import types +import uuid from collections.abc import Callable, Mapping, Sequence from copy import deepcopy +from datetime import datetime from typing import Final, List, TypeAlias, cast from unittest.mock import MagicMock, patch @@ -16,62 +20,49 @@ import pytest import respx import litellm +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + get_completion_messages, +) from litellm.litellm_core_utils.prompt_templates.factory import ( - BEDROCK_DOCUMENT_PLACEHOLDER_TEXT, _EMPTY_TEXT_PLACEHOLDER, + BEDROCK_DOCUMENT_PLACEHOLDER_TEXT, + THOUGHT_SIGNATURE_SEPARATOR, BedrockConverseMessagesProcessor, BedrockImageProcessor, _bedrock_converse_messages_pt, _bedrock_tools_pt, - _rename_duplicate_bedrock_document_names, _convert_to_bedrock_tool_call_invoke, - sanitize_anthropic_tool_use_id, _convert_to_bedrock_tool_call_result, + _rename_duplicate_bedrock_document_names, anthropic_messages_pt, - convert_to_anthropic_tool_result, - convert_to_gemini_tool_call_result, - convert_to_ollama_image, - encode_tool_call_id_with_signature, - function_call_prompt, - get_thought_signature_from_tool, - get_tool_calls_from_response, - make_valid_bedrock_tool_name, - ollama_pt, - parse_mime_type, - sanitize_messages_for_tool_calling, - THOUGHT_SIGNATURE_SEPARATOR, -) -from litellm.types.llms.openai import ChatCompletionAssistantMessage, ChatCompletionToolMessage -from litellm.types.utils import ChatCompletionMessageToolCall, Function -from litellm.utils import ( - _invalidate_model_cost_lowercase_map, - function_setup, - Rules, - validate_and_fix_openai_messages, -) -from datetime import datetime -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome -from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_pt, claude_2_1_pt, convert_generic_image_chunk_to_openai_image_obj, convert_to_anthropic_image_obj, convert_to_anthropic_tool_invoke, + convert_to_anthropic_tool_result, + convert_to_gemini_tool_call_result, + convert_to_ollama_image, convert_url_to_base64, create_anthropic_image_param, + encode_tool_call_id_with_signature, + function_call_prompt, + get_thought_signature_from_tool, + get_tool_calls_from_response, has_tool_with_name, llama_2_chat_pt, + make_valid_bedrock_tool_name, + ollama_pt, + parse_mime_type, prompt_factory, + response_schema_prompt, + sanitize_anthropic_tool_use_id, + sanitize_messages_for_tool_calling, ) -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, - get_completion_messages, ) -from litellm.llms.vertex_ai.gemini.transformation import gemini_convert_messages_with_history -from litellm.types.llms.bedrock import BedrockConverseReasoningContentBlock -from litellm.types.llms.openai import AllMessageValues -from litellm.litellm_core_utils.prompt_templates.factory import response_schema_prompt from litellm.llms.anthropic.completion.transformation import AnthropicTextConfig from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import AmazonInvokeConfig from litellm.llms.codestral.completion.handler import CodestralTextCompletion @@ -79,10 +70,19 @@ from litellm.llms.ollama.completion.transformation import OllamaConfig from litellm.llms.petals.completion import handler as petals_handler from litellm.llms.predibase.chat.transformation import PredibaseConfig from litellm.llms.sagemaker.completion.transformation import SagemakerConfig +from litellm.llms.vertex_ai.gemini.transformation import gemini_convert_messages_with_history from litellm.llms.vllm.completion import handler as vllm_handler from litellm.llms.watsonx.common_utils import _convert_watsonx_messages_core -from litellm.types.utils import ModelResponse, TextCompletionResponse - +from litellm.types.llms.bedrock import BedrockConverseReasoningContentBlock +from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionToolMessage +from litellm.types.utils import ChatCompletionMessageToolCall, Function, ModelResponse, TextCompletionResponse +from litellm.utils import ( + Rules, + _invalidate_model_cost_lowercase_map, + function_setup, + validate_and_fix_openai_messages, +) +from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome READ_FILE: Final[list[object]] = [{"name": "read_file", "description": "Read the contents of a file."}] USER_TURN: Final[dict[str, object]] = {"role": "user", "content": "Read the file."} @@ -3460,8 +3460,8 @@ def test_bedrock_tools_unpack_defs_no_oom_with_nested_refs(): This test creates a schema with multiple nested $defs that reference each other to verify the fix prevents memory explosion while still correctly resolving refs. """ - import sys import copy + import sys from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_tools_pt @@ -7852,7 +7852,7 @@ def test_anthropic_messages_pt_with_bash_tool_result_in_provider_specific_fields def test_parse_tool_call_arguments_valid_json(): """Test that valid JSON is parsed correctly.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, ) @@ -7862,7 +7862,7 @@ def test_parse_tool_call_arguments_valid_json(): def test_parse_tool_call_arguments_empty_input(): """Test that None/empty input returns empty dict.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, ) @@ -7872,7 +7872,7 @@ def test_parse_tool_call_arguments_empty_input(): def test_parse_tool_call_arguments_malformed_json(): """Test that malformed JSON raises ValueError with context.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, ) @@ -7918,7 +7918,7 @@ def test_convert_to_anthropic_tool_invoke_malformed_json(): def test_attempt_json_repair_missing_closing_brace(): """Repair JSON truncated with a missing closing brace (issue #22312).""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -7934,7 +7934,7 @@ def test_attempt_json_repair_missing_closing_brace(): def test_attempt_json_repair_missing_bracket_and_brace(): """Repair JSON truncated with both missing ] and }.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -7946,7 +7946,7 @@ def test_attempt_json_repair_missing_bracket_and_brace(): def test_attempt_json_repair_trailing_comma(): """Repair JSON with a trailing comma before missing close.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -7958,7 +7958,7 @@ def test_attempt_json_repair_trailing_comma(): def test_attempt_json_repair_returns_none_for_unterminated_string(): """Cannot repair an unterminated string — returns None.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -7967,7 +7967,7 @@ def test_attempt_json_repair_returns_none_for_unterminated_string(): def test_attempt_json_repair_returns_none_for_valid_json(): """Valid JSON has no unmatched brackets — returns None (no repair needed).""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -7976,7 +7976,7 @@ def test_attempt_json_repair_returns_none_for_valid_json(): def test_attempt_json_repair_returns_none_for_empty(): """Empty / whitespace input returns None.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -7986,7 +7986,7 @@ def test_attempt_json_repair_returns_none_for_empty(): def test_attempt_json_repair_interleaved_nesting(): """Repair JSON with interleaved {} and [] nesting.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -7998,7 +7998,7 @@ def test_attempt_json_repair_interleaved_nesting(): def test_attempt_json_repair_deeply_nested(): """Repair deeply nested truncated JSON.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( _attempt_json_repair, ) @@ -8010,7 +8010,7 @@ def test_attempt_json_repair_deeply_nested(): def test_parse_tool_call_arguments_whitespace_only(): """Whitespace-only input returns empty dict.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, ) @@ -8020,7 +8020,7 @@ def test_parse_tool_call_arguments_whitespace_only(): def test_parse_tool_call_arguments_non_object_json(): """Non-object JSON (list, string, number) is returned as-is (no wrapping).""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, ) @@ -8030,7 +8030,7 @@ def test_parse_tool_call_arguments_non_object_json(): def test_parse_tool_call_arguments_repairs_truncated_json(): """parse_tool_call_arguments should repair truncated JSON instead of raising.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, ) @@ -8041,7 +8041,7 @@ def test_parse_tool_call_arguments_repairs_truncated_json(): def test_parse_tool_call_arguments_still_raises_for_unrepairable(): """parse_tool_call_arguments raises ValueError when repair also fails.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, ) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py index d7696c00dee..34d525946b3 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py @@ -12,13 +12,15 @@ from typing import Final import pytest import litellm -from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( CONVERTED_SYSTEM_NOTE, anthropic_system_blocks, place_mid_conversation_system, split_leading_system_run, ) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + encrypted_reasoning_signature, +) def _roles(messages: object) -> list[str]: diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_prompt_cache_breakpoint_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_prompt_cache_breakpoint_utils.py new file mode 100644 index 00000000000..9dd50368529 --- /dev/null +++ b/tests/unit/litellm_core_utils/prompt_templates/test_prompt_cache_breakpoint_utils.py @@ -0,0 +1,113 @@ +from typing import Final + +import pytest + +from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( + cache_control_as_prompt_cache_breakpoint, + prompt_cache_breakpoint_for_wire, + with_prompt_cache_breakpoint, +) + +_MALFORMED_PROMPT_CACHE_BREAKPOINTS: Final = ( + "yes", + 1, + ["explicit"], + "", + {}, + {"mode": "bogus"}, + {"mode": "bogus", "ttl": "30m"}, + {1: "explicit"}, +) + + +_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS: Final = ( + "string", + "int", + "list", + "empty-string", + "empty-object", + "bogus-mode", + "bogus-mode-with-ttl", + "non-string-key", +) + + +_VALID_PROMPT_CACHE_BREAKPOINTS: Final = ({"mode": "explicit"}, {"mode": "explicit", "ttl": "30m"}) + + +@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep")) +def test_prompt_cache_breakpoint_for_wire_passes_a_missing_marker_through(drop_params: bool) -> None: + assert prompt_cache_breakpoint_for_wire(None, drop_params) is None + + +@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS) +def test_prompt_cache_breakpoint_for_wire_drops_a_malformed_marker_under_drop_params(marker: object) -> None: + assert prompt_cache_breakpoint_for_wire(marker, True) is None + + +@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS) +def test_prompt_cache_breakpoint_for_wire_forwards_a_malformed_marker_without_drop_params(marker: object) -> None: + assert prompt_cache_breakpoint_for_wire(marker, False) is marker + + +@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep")) +@pytest.mark.parametrize("marker", _VALID_PROMPT_CACHE_BREAKPOINTS, ids=("explicit", "explicit-30m")) +def test_prompt_cache_breakpoint_for_wire_keeps_a_valid_marker(marker: dict[str, str], drop_params: bool) -> None: + assert prompt_cache_breakpoint_for_wire(marker, drop_params) == marker + + +def test_prompt_cache_breakpoint_for_wire_strips_an_unknown_key_under_drop_params() -> None: + assert prompt_cache_breakpoint_for_wire({"mode": "explicit", "note": "kept"}, True) == {"mode": "explicit"} + + +@pytest.mark.parametrize( + "marker", + ({"mode": "explicit", "ttl": "1h"}, {"mode": "explicit", "ttl": "1h", "note": "kept"}), + ids=("bad-ttl", "bad-ttl-and-unknown-key"), +) +def test_prompt_cache_breakpoint_for_wire_drops_an_unsupported_ttl_and_keeps_the_mode_under_drop_params( + marker: dict[str, str], +) -> None: + assert prompt_cache_breakpoint_for_wire(marker, True) == {"mode": "explicit"} + assert prompt_cache_breakpoint_for_wire(marker, False) is marker + + +def test_cache_control_is_replaced_by_an_explicit_prompt_cache_breakpoint_on_the_same_block() -> None: + marked: Final = {"type": "text", "text": "rules", "cache_control": {"type": "ephemeral", "ttl": "1h"}} + + assert cache_control_as_prompt_cache_breakpoint(marked) == { + "type": "text", + "text": "rules", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + + +def test_a_block_without_cache_control_is_returned_untouched() -> None: + unmarked: Final = {"type": "text", "text": "rules"} + + assert cache_control_as_prompt_cache_breakpoint(unmarked) is unmarked + + +def test_a_breakpoint_the_client_wrote_survives_a_message_level_mark_ttl_included() -> None: + block: Final = { + "type": "input_text", + "text": "rules", + "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}, + } + + assert with_prompt_cache_breakpoint(block, {"mode": "explicit"}) is block + + +def test_a_breakpoint_the_client_wrote_survives_a_per_part_cache_control() -> None: + both: Final = { + "type": "input_text", + "text": "rules", + "cache_control": {"type": "ephemeral"}, + "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}, + } + + assert cache_control_as_prompt_cache_breakpoint(both) == { + "type": "input_text", + "text": "rules", + "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}, + } diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_reasoning_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_reasoning_utils.py new file mode 100644 index 00000000000..840497777ad --- /dev/null +++ b/tests/unit/litellm_core_utils/prompt_templates/test_reasoning_utils.py @@ -0,0 +1,171 @@ +import base64 +import copy +from typing import Final + +import pytest + +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + ENCRYPTED_REASONING_SIGNATURE_PREFIX, + converse_redacted_content, + encrypted_content_from_signature, + encrypted_reasoning_signature, + is_encrypted_reasoning_block, + native_redacted_thinking_data, + responses_reasoning_items_from_thinking_blocks, + strip_encrypted_reasoning_from_messages, +) + + +class TestEncryptedReasoningReplay: + """Regression for https://github.com/BerriAI/litellm/issues/40288.""" + + def test_signature_round_trips_the_encrypted_content(self): + assert encrypted_content_from_signature(encrypted_reasoning_signature("gAAAA_bytes")) == "gAAAA_bytes" + + @pytest.mark.parametrize( + "signature", [None, "", "ErcBCkgIValidAnthropicSignature", "litellm_encrypted_reasoning:", 7] + ) + def test_anything_else_is_not_encrypted_content(self, signature): + assert encrypted_content_from_signature(signature) is None + + def test_encrypted_thinking_block_replays_its_own_item(self): + items = responses_reasoning_items_from_thinking_blocks( + [{"type": "thinking", "thinking": "Plan.", "signature": encrypted_reasoning_signature("gAAAA_1")}] + ) + assert items == ( + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Plan."}], + "encrypted_content": "gAAAA_1", + }, + ) + + def test_encrypted_redacted_block_replays_with_an_empty_summary(self): + items = responses_reasoning_items_from_thinking_blocks( + [{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_1")}] + ) + assert items == ({"type": "reasoning", "summary": [], "encrypted_content": "gAAAA_1"},) + + def test_plain_blocks_collapse_into_one_summary_item_around_encrypted_ones(self): + items = responses_reasoning_items_from_thinking_blocks( + [ + {"type": "thinking", "thinking": "A.", "signature": None}, + {"type": "thinking", "thinking": "B.", "signature": ""}, + {"type": "thinking", "thinking": "C.", "signature": encrypted_reasoning_signature("gAAAA_c")}, + {"type": "redacted_thinking", "data": "anthropic-minted-opaque-data"}, + {"type": "thinking", "thinking": "D."}, + ] + ) + assert items == ( + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "A."}, {"type": "summary_text", "text": "B."}], + }, + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "C."}], "encrypted_content": "gAAAA_c"}, + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "D."}]}, + ) + assert all("id" not in item for item in items) + + def test_blocks_without_text_or_encrypted_content_produce_nothing(self): + assert responses_reasoning_items_from_thinking_blocks([{"type": "thinking", "thinking": ""}]) == () + assert responses_reasoning_items_from_thinking_blocks([]) == () + + @pytest.mark.parametrize( + ("block", "expected"), + [ + ({"type": "thinking", "thinking": "x", "signature": encrypted_reasoning_signature("g")}, True), + ({"type": "redacted_thinking", "data": encrypted_reasoning_signature("g")}, True), + ({"type": "thinking", "thinking": "x", "signature": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True), + ({"type": "redacted_thinking", "data": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True), + ({"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}, False), + ({"type": "redacted_thinking", "data": "EmwKAhgBEgy"}, False), + ({"type": "text", "text": encrypted_reasoning_signature("g")}, False), + ("not a block", False), + ], + ) + def test_is_encrypted_reasoning_block(self, block, expected): + assert is_encrypted_reasoning_block(block) is expected + + def test_strip_drops_every_bridge_tagged_block_and_leaves_no_unsigned_thinking_behind(self): + assistant_content = [ + {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, + {"type": "thinking", "thinking": "packed by the bridge", "signature": encrypted_reasoning_signature("g1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("g2")}, + {"type": "thinking", "thinking": "", "signature": encrypted_reasoning_signature("g3")}, + {"type": "text", "text": "answer"}, + ] + messages = [ + {"role": "user", "content": "question"}, + {"role": "assistant", "content": assistant_content}, + {"role": "user", "content": [{"type": "text", "text": "follow-up"}]}, + ] + + strip_encrypted_reasoning_from_messages(messages) + + assert messages[1]["content"] is assistant_content + assert assistant_content == [ + {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, + {"type": "text", "text": "answer"}, + ] + assert all(block["signature"] for block in assistant_content if block["type"] == "thinking") + assert messages[0] == {"role": "user", "content": "question"} + assert messages[2] == {"role": "user", "content": [{"type": "text", "text": "follow-up"}]} + + def test_strip_uses_predicate_to_keep_selected_encrypted_blocks(self): + kept_signature = encrypted_reasoning_signature("keep") + stripped_signature = encrypted_reasoning_signature("strip") + content = [ + {"type": "thinking", "thinking": "keep", "signature": kept_signature}, + {"type": "thinking", "thinking": "strip", "signature": stripped_signature}, + ] + messages = [{"role": "assistant", "content": content}] + + strip_encrypted_reasoning_from_messages(messages, should_strip=lambda block: block.get("thinking") == "strip") + + assert messages[0]["content"] is content + assert content == [{"type": "thinking", "thinking": "keep", "signature": kept_signature}] + + @pytest.mark.parametrize( + "messages", + [ + "not a list", + None, + [{"role": "user", "content": None}], + [{"role": "user", "content": "plain string"}], + ["not a message"], + [{"role": "assistant", "content": [{"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}]}], + ], + ) + def test_strip_leaves_history_without_bridge_reasoning_untouched(self, messages): + before = copy.deepcopy(messages) + + strip_encrypted_reasoning_from_messages(messages) + + assert messages == before + + +_NATIVE_REDACTED_DATA: Final = "EmwKAhgBEgy3va3pzix/LafPsn4aDFIT2Xlxh0L5L8rLVyIwxtE3rAFBa8cr3qpP" + + +_CONVERSE_REDACTED_DATA: Final = base64.b64encode(_NATIVE_REDACTED_DATA.encode()).decode() + + +@pytest.mark.parametrize( + "data, native", + [ + pytest.param(_NATIVE_REDACTED_DATA, _NATIVE_REDACTED_DATA, id="claude-native"), + pytest.param(_CONVERSE_REDACTED_DATA, _NATIVE_REDACTED_DATA, id="converse-wrapped-legacy-history"), + pytest.param( + encrypted_reasoning_signature("gAAAA_1"), encrypted_reasoning_signature("gAAAA_1"), id="openai-encrypted" + ), + pytest.param("", "", id="blank"), + pytest.param(base64.b64encode(b"not base64!").decode(), base64.b64encode(b"not base64!").decode(), id="b64-once"), + ], +) +def test_native_redacted_thinking_data_only_unwraps_the_converse_base64_layer(data: str, native: str) -> None: + assert native_redacted_thinking_data(data) == native + + +@pytest.mark.parametrize("data", [_NATIVE_REDACTED_DATA, _CONVERSE_REDACTED_DATA], ids=["native", "legacy-wrapped"]) +def test_converse_redacted_content_wraps_claude_native_data_exactly_once(data: str) -> None: + assert converse_redacted_content(data) == _CONVERSE_REDACTED_DATA diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_tool_call_json_repair.py b/tests/unit/litellm_core_utils/prompt_templates/test_tool_call_json_repair.py new file mode 100644 index 00000000000..75f4e686137 --- /dev/null +++ b/tests/unit/litellm_core_utils/prompt_templates/test_tool_call_json_repair.py @@ -0,0 +1,117 @@ +import json + +import pytest + +from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( + parse_tool_call_arguments, + salvage_concatenated_tool_arguments, + split_concatenated_json_objects, +) + + +def test_split_concatenated_json_single_object(): + """A single valid JSON object is returned as a one-element list.""" + result = split_concatenated_json_objects('{"location": "Boston"}') + assert result == [{"location": "Boston"}] + + +def test_split_concatenated_json_multiple_objects(): + """ + Multiple JSON objects concatenated without separators are split correctly. + This is the exact pattern from issue #20543 where Bedrock Claude Sonnet 4.5 + returns concatenated JSON in a single tool call arguments string. + """ + raw = ( + '{"command": ["curl", "-i", "http://localhost:9009"]}' + '{"command": ["curl", "-i", "http://localhost:9009/robots.txt"]}' + '{"command": ["curl", "-i", "http://localhost:9009/sitemap.xml"]}' + ) + result = split_concatenated_json_objects(raw) + assert len(result) == 3 + assert result[0] == {"command": ["curl", "-i", "http://localhost:9009"]} + assert result[1] == {"command": ["curl", "-i", "http://localhost:9009/robots.txt"]} + assert result[2] == {"command": ["curl", "-i", "http://localhost:9009/sitemap.xml"]} + + +def test_split_concatenated_json_with_whitespace(): + """Objects separated by whitespace are handled correctly.""" + raw = '{"a": 1} {"b": 2}\n{"c": 3}' + result = split_concatenated_json_objects(raw) + assert len(result) == 3 + assert result[0] == {"a": 1} + assert result[1] == {"b": 2} + assert result[2] == {"c": 3} + + +def test_split_concatenated_json_empty_string(): + """Empty or whitespace-only strings return an empty list.""" + assert split_concatenated_json_objects("") == [] + assert split_concatenated_json_objects(" ") == [] + + +def test_split_concatenated_json_non_dict_value(): + """Non-dict JSON values (e.g. arrays, strings) are replaced with {}.""" + result = split_concatenated_json_objects("[1, 2, 3]") + assert result == [{}] + + +def test_split_concatenated_json_wholly_invalid_returns_empty(): + """ + Wholly unparseable JSON degrades to an empty list instead of raising. + + Regression for https://github.com/BerriAI/litellm/issues/18667: a raise + here propagated out of `_convert_to_bedrock_tool_call_invoke` and turned + every replayed conversation into a 500. + """ + assert split_concatenated_json_objects("not json at all") == [] + + +def test_split_concatenated_json_malformed_object_returns_empty(): + """ + A single malformed object (missing comma between keys) degrades to an + empty list rather than raising `Expecting ',' delimiter`. + """ + assert split_concatenated_json_objects('{"location": "Boston" "unit": "celsius"}') == [] + + +def test_split_concatenated_json_salvages_prefix_before_truncated_tail(): + """ + Complete objects parsed before an unparseable/truncated tail are kept; + only the bad tail is discarded. + """ + result = split_concatenated_json_objects('{"a": 1}{"b": 2}{"c":') + assert result == [{"a": 1}, {"b": 2}] + + +def test_parse_tool_call_arguments_rejects_concatenated_json() -> None: + with pytest.raises(ValueError, match="Failed to parse tool call arguments"): + parse_tool_call_arguments('{"a":1}{"b":2}') + + +def _distinct_json_objects(count: int) -> str: + return "".join(json.dumps({"n": index}, separators=(",", ":")) for index in range(count)) + + +@pytest.mark.parametrize( + ("raw", "expected"), + ( + ('{"a":1}{"b":2}', ({"a": 1}, {"b": 2})), + ('{"a":1}{"a":1}{"a":1}', ({"a": 1},)), + ('{"a":1}{"a":1}{"b":2}', ({"a": 1}, {"a": 1}, {"b": 2})), + (_distinct_json_objects(8), tuple({"n": index} for index in range(8))), + (_distinct_json_objects(9), ()), + (_distinct_json_objects(9) + " junk", ()), + ('{"a":1}' * 7 + '{"b":2}', tuple({"a": 1} for _ in range(7)) + ({"b": 2},)), + ('{"a":1}' * 8 + '{"b":2}', ()), + ('{"a":1}' * 5000, ({"a": 1},)), + ('{"a":1}' * 20, ({"a": 1},)), + ('{"a":1}{"b":', ()), + ('0{"x":1}', ()), + ('{"x":1}0', ()), + ('[1]{"x":1}', ()), + ('{"a":1}{"b":2}}', ()), + ('{"a":1} junk', ()), + ), +) +def test_salvage_concatenated_tool_arguments(raw: str, expected: tuple[dict[str, object], ...]) -> None: + assert salvage_concatenated_tool_arguments(raw) == expected diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_tpm_media_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_tpm_media_utils.py new file mode 100644 index 00000000000..81a688e71e2 --- /dev/null +++ b/tests/unit/litellm_core_utils/prompt_templates/test_tpm_media_utils.py @@ -0,0 +1,146 @@ +import base64 +import copy +import json +from collections.abc import Mapping +from typing import Final + +import pytest + +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( + split_embedding_media_for_tpm, + split_media_for_tpm, +) + + +@pytest.mark.parametrize( + "part,projected,media", + [ + ( + {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "high"}}, + {"type": "image_url", "image_url": {"url": "", "detail": "high"}}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "high"}}, + ), + ( + {"type": "input_image", "image_url": "https://example.com/image.png", "detail": "low"}, + {"type": "input_image", "image_url": "", "detail": "low"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.png", "detail": "low"}}, + ), + ( + {"type": "computer_screenshot", "image_url": "data:image/png;base64,IMAGE"}, + {"type": "computer_screenshot", "image_url": ""}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, + ), + ( + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "IMAGE"}}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, + ), + ( + {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}, + {"type": "input_audio", "input_audio": {"data": "", "format": "wav"}}, + {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}, + ), + ( + {"type": "file", "file": {"file_data": "IMAGE", "format": "image/png", "filename": "image.png"}}, + {"type": "file", "file": {"file_data": "", "format": "image/png", "filename": "image.png"}}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, + ), + ( + {"type": "input_file", "file_data": "data:audio/wav;base64,AUDIO", "filename": "audio.wav"}, + {"type": "input_file", "file_data": "", "filename": "audio.wav"}, + {"type": "input_audio", "input_audio": {"data": "AUDIO", "format": "wav"}}, + ), + ], +) +def test_tpm_projection_blanks_only_declared_media_transport( + part: Mapping[str, object], projected: Mapping[str, object], media: Mapping[str, object], +) -> None: + text: Final = {"type": "text", "text": "data:image/png;base64,IMAGE"} + request: Final = [{"role": "user", "content": [text, {**part, "cache_control": {"type": "ephemeral"}}]}] + original: Final = copy.deepcopy(request) + assert split_media_for_tpm(request) == ( + [{"role": "user", "content": [text, {**projected, "cache_control": {"type": "ephemeral"}}]}], + (media,), + ) + assert request == original + + +def test_tpm_projection_retains_nested_and_native_history_around_media() -> None: + image: Final = {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "high"}} + empty_image: Final = {"type": "image_url", "image_url": {"url": "", "detail": "high"}} + document: Final = {"type": "document", "title": "keep", "source": {"type": "content", "content": [image]}} + empty_document: Final = {**document, "source": {"type": "content", "content": [empty_image]}} + tool: Final = {"type": "tool_result", "tool_use_id": "call", "content": [document, {"type": "text", "text": "keep"}]} + empty_tool: Final = {**tool, "content": [empty_document, {"type": "text", "text": "keep"}]} + request: Final = [ + {"role": "user", "content": [tool]}, + {"type": "function_call_output", "call_id": "call", "output": [image]}, + {"type": "custom_tool_call_output", "call_id": "custom", "output": [image]}, + {"type": "computer_call_output", "call_id": "computer", "output": { + "type": "computer_screenshot", "image_url": "data:image/png;base64,IMAGE", + }}, + {"type": "image_generation_call", "id": "ig", "result": "IMAGE", "status": "completed"}, + {"type": "code_interpreter_call", "id": "ci", "code": "keep()", "outputs": [ + {"type": "logs", "logs": "keep"}, {"type": "image", "url": "https://example.com/image.png"}, + ]}, + ] + original: Final = copy.deepcopy(request) + projected, media = split_media_for_tpm(request) + assert projected == [ + {"role": "user", "content": [empty_tool]}, + {"type": "function_call_output", "call_id": "call", "output": [empty_image]}, + {"type": "custom_tool_call_output", "call_id": "custom", "output": [empty_image]}, + {"type": "computer_call_output", "call_id": "computer", "output": { + "type": "computer_screenshot", "image_url": "", + }}, + {"type": "image_generation_call", "id": "ig", "result": "", "status": "completed"}, + {"type": "code_interpreter_call", "id": "ci", "code": "keep()", "outputs": [ + {"type": "logs", "logs": "keep"}, {"type": "image", "url": ""}, + ]}, + ] + assert media == ( + image, image, image, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,IMAGE", "detail": "auto"}}, + {"type": "image_url", "image_url": {"url": "data:application/octet-stream;base64,IMAGE", "detail": "auto"}}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.png", "detail": "auto"}}, + ) + assert request == original + + +def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None: + image: Final = {"type": "input_image", "image_url": "data:image/png;base64,OPAQUE"} + request: Final = [ + {"role": "user", "content": json.dumps(image)}, + {"type": "custom_tool_call_output", "call_id": "call", "output": json.dumps(image)}, + {"type": "function_call", "call_id": "call", "name": "keep", "arguments": {"content": [image]}}, + {"type": "file_search_call", "results": [{"text": "keep", "content": [image]}]}, + {"type": "image_generation_call", "id": "ig", "status": "in_progress", "result": None}, + {"role": "user", "content": [ + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "PDF"}}, + {"type": "document", "source": {"type": "text", "data": json.dumps(image)}}, + {"type": "file", "file": {"format": "application/pdf", "file_data": "PDF"}}, + {"type": "text", "text": "keep", "metadata": {"content": [image]}}, + ]}, + ] + projected, media = split_media_for_tpm(request) + assert projected is request + assert media == () + + +@pytest.mark.parametrize("mime", ["application/pdf", "video/mp4", "audio/wav", "image/png"]) +def test_embedding_media_split_holds_at_most_one_copy_of_a_large_payload(mime: str) -> None: + import gc + import tracemalloc + + uri: Final = f"data:{mime};base64," + base64.b64encode(bytes(3 * 1024 * 1024)).decode("ascii") + split_embedding_media_for_tpm(f"data:{mime};base64,AAAA") + gc.collect() + tracemalloc.start() + try: + _, media = split_embedding_media_for_tpm(uri) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert media + assert peak < 1.5 * len(uri) diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 3a4231443dd..95bc8af7270 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -8,9 +8,9 @@ from typing import Final from unittest.mock import MagicMock, patch import httpx -from httpx import Headers import pytest import respx +from httpx import Headers from pydantic import BaseModel, ConfigDict import litellm @@ -24,20 +24,22 @@ from litellm.constants import ( RESPONSE_FORMAT_TOOL_NAME, RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) -from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + encrypted_reasoning_signature, +) from litellm.llms.anthropic.chat import ModelResponseIterator -from litellm.llms.anthropic.common_utils import AnthropicError, process_anthropic_headers from litellm.llms.anthropic.chat.transformation import AnthropicConfig -from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema +from litellm.llms.anthropic.common_utils import AnthropicError, process_anthropic_headers from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig -from litellm.llms.bedrock.claude_platform.transformation import BedrockClaudePlatformConfig +from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) +from litellm.llms.bedrock.claude_platform.transformation import BedrockClaudePlatformConfig from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( VertexAIAnthropicConfig, ) @@ -9067,10 +9069,11 @@ async def test_anthropic_api_max_completion_tokens(model: str): def test_anthropic_tool_cache_control(): - from litellm.utils import return_raw_request - from litellm.types.utils import CallTypes import json + from litellm.types.utils import CallTypes + from litellm.utils import return_raw_request + tool_content = "Result: 4. " * 1000 messages = [ {"role": "user", "content": "Calculate 2+2"}, diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index cc366dcde4c..fb4f19a862c 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -11,17 +11,16 @@ import pytest import respx import litellm - - - from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_PLACEHOLDER, - encrypted_reasoning_signature, ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, _bedrock_converse_messages_pt, ) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + encrypted_reasoning_signature, +) from litellm.llms.anthropic.messages.adapters.transformation import ( OPENAI_MAX_TOOL_NAME_LENGTH, AnthropicAdapter, @@ -3302,7 +3301,6 @@ def test_translate_openai_response_to_anthropic_input_tokens_no_cache(): """ Regression test: input_tokens should equal prompt_tokens when there are no cached tokens. """ - from litellm.types.utils import PromptTokensDetailsWrapper # Create OpenAI format response without cached tokens usage = Usage( diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py index cd6d2d9be96..4b52f7fbef1 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py @@ -1,4 +1,4 @@ -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, ) from litellm.llms.anthropic.messages.native.transformation import ( diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py index 4626f76c8fc..ffda84f32b7 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -19,7 +19,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../. import litellm from litellm.exceptions import MidStreamFallbackError from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, ) from litellm.llms.anthropic.messages.native.streaming_iterator import aclose_superseded_attempt diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index ece0cf5d423..dbb8c925fa0 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -20,6 +20,8 @@ from litellm.constants import ( from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, +) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, ) from litellm.llms.anthropic.messages.responses_adapters.transformation import ( diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index 99e6438c723..c97585e02c2 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -1745,7 +1745,7 @@ class TestAnthropicThinkingSignatureSelfHeal: def test_strip_keeps_encrypted_reasoning_blocks_for_the_responses_bridge(self): """The /v1/messages handler runs this before dispatch, so the bridge must still see the replay.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, ) from litellm.llms.anthropic.common_utils import ( @@ -1766,7 +1766,7 @@ class TestAnthropicThinkingSignatureSelfHeal: def test_strip_encrypted_reasoning_drops_only_the_bridge_tagged_blocks(self): """A session resumed on an Anthropic model replays reasoning only OpenAI can verify.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( encrypted_reasoning_signature, ) from litellm.llms.anthropic.common_utils import ( diff --git a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py index f93968eb2e6..8571a0c3cbc 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -1,7 +1,11 @@ -import asyncio, httpx, importlib, json, os +import asyncio +import importlib +import json +import os import sys from typing import Final +import httpx import pytest from pydantic import TypeAdapter @@ -9,19 +13,21 @@ sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) ) -import litellm -from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_BOUNDARY -from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config -from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig -from litellm.utils import _invalidate_model_cost_lowercase_map, get_optional_params from datetime import datetime -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.router import Router +from unittest.mock import AsyncMock + from openai.types.chat import ChatCompletionMessage from openai.types.chat.chat_completion import ChatCompletion, Choice from respx import MockRouter + +import litellm +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_BOUNDARY +from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config +from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig +from litellm.router import Router +from litellm.utils import _invalidate_model_cost_lowercase_map, get_optional_params from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome -from unittest.mock import AsyncMock _MAPPED_PARAMS: Final = TypeAdapter(dict[str, object]) _SUPPORTED_PARAMS: Final = TypeAdapter(list[str]) @@ -505,7 +511,7 @@ async def test_azure_gpt5_async_transform_drops_tool_choice_without_tools() -> N def test_transform_request_strips_litellm_format_from_managed_file_id(): import base64 - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( update_messages_with_model_file_ids, ) diff --git a/tests/unit/llms/test_file_content_block.py b/tests/unit/llms/test_file_content_block.py index 4ab4ef8b857..297b3b6f7d6 100644 --- a/tests/unit/llms/test_file_content_block.py +++ b/tests/unit/llms/test_file_content_block.py @@ -19,14 +19,16 @@ import pytest import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( - get_file_ids_from_messages, migrate_file_to_image_url, - update_messages_with_model_file_ids, ) from litellm.litellm_core_utils.prompt_templates.factory import ( BedrockConverseMessagesProcessor, anthropic_process_openai_file_message, ) +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_file_ids_from_messages, + update_messages_with_model_file_ids, +) from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.vertex_ai.gemini.transformation import ( diff --git a/tests/unit/llms/test_file_search_responses.py b/tests/unit/llms/test_file_search_responses.py index b2cbcd4a6dd..424aaad1fd1 100644 --- a/tests/unit/llms/test_file_search_responses.py +++ b/tests/unit/llms/test_file_search_responses.py @@ -13,13 +13,13 @@ Coverage: import base64 import logging -from typing import Any, Dict, List, Optional from importlib import import_module +from typing import Any, Dict, List, Optional from unittest.mock import AsyncMock, MagicMock, patch import pytest -from litellm.litellm_core_utils.prompt_templates.common_utils import ( +from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( _decode_vector_store_ids_in_tools, update_responses_tools_with_model_file_ids, ) @@ -476,6 +476,7 @@ class TestManagedFilesVectorStoreAccess: from litellm_enterprise.proxy.hooks.managed_files import ( PROXY_LiteLLMManagedFiles as ManagedFiles, ) + from litellm.proxy._types import CallTypes # If call_type is acompletion, the vector_store check branch isn't reached. diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 2c8c7520c38..aec68f1fa45 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -1980,7 +1980,6 @@ def test_file_data_field_order(): def test_file_data_field_order_gcs_urls(): """Test that GCS URLs also maintain correct field order.""" - import json from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media @@ -2066,7 +2065,7 @@ def test_extract_file_data_with_path_object(): import tempfile from pathlib import Path - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( extract_file_data, ) @@ -2105,7 +2104,7 @@ def test_extract_file_data_with_pathlib_path(): import tempfile from pathlib import Path - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( extract_file_data, ) @@ -2129,7 +2128,7 @@ def test_extract_file_data_with_pathlib_path(): def test_extract_file_data_with_tuple_format(): """Test that tuple format (with explicit content_type) still works correctly.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( extract_file_data, ) @@ -2152,7 +2151,7 @@ def test_extract_file_data_fallback_to_octet_stream(): import tempfile from pathlib import Path - from litellm.litellm_core_utils.prompt_templates.common_utils import ( + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( extract_file_data, ) diff --git a/tests/unit/llms/vertex_ai/test_vertex.py b/tests/unit/llms/vertex_ai/test_vertex.py index 7e0baaf9e10..9503182cad7 100644 --- a/tests/unit/llms/vertex_ai/test_vertex.py +++ b/tests/unit/llms/vertex_ai/test_vertex.py @@ -6,9 +6,9 @@ import sys from types import SimpleNamespace from typing import Final -from dotenv import load_dotenv import httpx import respx +from dotenv import load_dotenv from google.oauth2.credentials import Credentials import litellm.litellm_core_utils @@ -23,7 +23,6 @@ import pytest import litellm from litellm import Router, get_optional_params from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media -from litellm.types.llms.vertex_ai import BlobType def encode_image_to_base64(image_path): @@ -110,7 +109,6 @@ def test_completion_pydantic_obj_2(): def test_build_vertex_schema(): - import json from litellm.llms.vertex_ai.common_utils import build_vertex_schema @@ -1278,7 +1276,9 @@ def test_process_gemini_media(): def test_get_image_mime_type_from_url(): """Test MIME type inference for remote media URLs""" - from litellm.litellm_core_utils.prompt_templates.common_utils import get_image_mime_type_from_url + from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( + get_image_mime_type_from_url, + ) # Test JPEG images assert ( @@ -1350,7 +1350,7 @@ def test_vertex_embedding_url(model, expected_url): assert endpoint == "predict" -from unittest.mock import Mock, patch +from unittest.mock import patch import pytest