mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-16 04:29:31 +00:00
fix(anthropic): cache tool result screenshots on the /v1/messages to Responses bridge (#46324)
* fix(anthropic): put a tool_result cache breakpoint on its hoisted images on the Responses bridge A marked tool_result whose images get moved into a trailing user message now carries the breakpoint on the last moved image instead of the function_call_output, so the prefix OpenAI caches includes the images * test(anthropic): cover a marked tool_result with two screenshots on the Responses bridge * fix(anthropic): keep every tool result screenshot inside the breakpoint that covers it on the Responses bridge A marked tool_result's screenshots now go right after its output, so earlier unmarked screenshots land inside the breakpoint and later tool outputs stay after it. A cache_control mark on a block inside a tool_result is read too. * fix(anthropic): cover only the screenshots up to a mark inside a tool result, and drop raw breakpoints under drop_params A cache mark on a text block inside a tool result now covers that result only up to the marked block, the way Anthropic caches it, so screenshots after the marked text stay outside the breakpoint. A mark on the tool result itself still covers every screenshot in it A deployment that reads no prompt cache breakpoints now drops raw prompt_cache_breakpoint marks, top-level and inside tool results, when drop_params is on, and still forwards them as sent without it * fix(anthropic): keep raw breakpoints under drop_params for models that take them * docs(anthropic): say a mark inside a tool result covers its whole text and documents --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: yuneng-jiang <yuneng@berri.ai>
This commit is contained in:
parent
14e820ef7c
commit
b37eac29f0
5 changed files with 474 additions and 41 deletions
|
|
@ -4,13 +4,17 @@ Handler for the Anthropic v1/messages -> OpenAI Responses API path.
|
|||
Used when the target model is an OpenAI or Azure model.
|
||||
"""
|
||||
|
||||
from collections.abc import AsyncIterator, Coroutine, Mapping, Sequence
|
||||
from collections.abc import AsyncIterator, Callable, Coroutine, Mapping, Sequence
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
supports_openai_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
|
||||
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
|
||||
from litellm.types.llms.anthropic import (
|
||||
|
|
@ -31,6 +35,7 @@ from .streaming_iterator import AnthropicResponsesStreamWrapper
|
|||
from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
|
||||
|
||||
AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]]
|
||||
BlockRewrite: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]]
|
||||
_BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
|
||||
|
||||
_ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
|
||||
|
|
@ -60,10 +65,46 @@ def _reads_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, ob
|
|||
)
|
||||
|
||||
|
||||
def _blocks_with_prompt_cache_breakpoints(content: object) -> Sequence[Mapping[str, object]]:
|
||||
"""The content's blocks with each ``cache_control`` mark rewritten, empty when it is not a list of blocks."""
|
||||
def _model_takes_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, object]) -> bool:
|
||||
provider: Final = forwarded_kwargs.get("custom_llm_provider")
|
||||
provider_name: Final = provider if isinstance(provider, str) else None
|
||||
base_model: Final = forwarded_kwargs.get("base_model")
|
||||
return supports_openai_prompt_cache_breakpoint(model, provider_name) or (
|
||||
isinstance(base_model, str)
|
||||
and bool(base_model)
|
||||
and supports_openai_prompt_cache_breakpoint(base_model, provider_name)
|
||||
)
|
||||
|
||||
|
||||
def _without_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"}
|
||||
|
||||
|
||||
def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, object]) -> BlockRewrite | None:
|
||||
"""Client cache marks become breakpoints for a deployment that reads them, and a model that takes no breakpoints
|
||||
drops raw ones under drop_params."""
|
||||
if _reads_prompt_cache_breakpoint(model, forwarded_kwargs):
|
||||
return cache_control_as_prompt_cache_breakpoint
|
||||
if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs):
|
||||
return None
|
||||
if normalize_drop_params(forwarded_kwargs.get("drop_params")) is True or litellm.drop_params is True:
|
||||
return _without_prompt_cache_breakpoint
|
||||
return None
|
||||
|
||||
|
||||
def _rewritten_block(block: Mapping[str, object], rewrite: BlockRewrite) -> Mapping[str, object]:
|
||||
rewritten: Final = rewrite(block)
|
||||
if rewritten.get("type") != "tool_result":
|
||||
return rewritten
|
||||
inner_blocks: Final = _rewritten_blocks(rewritten.get("content"), rewrite)
|
||||
return {**rewritten, "content": list(inner_blocks)} if inner_blocks else rewritten
|
||||
|
||||
|
||||
def _rewritten_blocks(content: object, rewrite: BlockRewrite) -> Sequence[Mapping[str, object]]:
|
||||
"""The content's blocks each rewritten, a tool_result's inner blocks included, empty when it is not a list of
|
||||
blocks."""
|
||||
try:
|
||||
return [cache_control_as_prompt_cache_breakpoint(block) for block in _BLOCK_LIST.validate_python(content)]
|
||||
return [_rewritten_block(block, rewrite) for block in _BLOCK_LIST.validate_python(content)]
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
|
@ -92,22 +133,19 @@ def _build_responses_kwargs(
|
|||
Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses().
|
||||
"""
|
||||
forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
|
||||
reads_prompt_cache_breakpoint: Final = _reads_prompt_cache_breakpoint(model, forwarded_kwargs)
|
||||
block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs)
|
||||
request_messages: Final[AnthropicRequestMessages] = (
|
||||
[
|
||||
{
|
||||
**message,
|
||||
"content": list(_blocks_with_prompt_cache_breakpoints(message.get("content")))
|
||||
or message.get("content"),
|
||||
"content": list(_rewritten_blocks(message.get("content"), block_rewrite)) or message.get("content"),
|
||||
}
|
||||
for message in messages
|
||||
]
|
||||
if reads_prompt_cache_breakpoint
|
||||
if block_rewrite
|
||||
else messages
|
||||
)
|
||||
request_system: Final = (
|
||||
(list(_blocks_with_prompt_cache_breakpoints(system)) or system) if reads_prompt_cache_breakpoint else system
|
||||
)
|
||||
request_system: Final = (list(_rewritten_blocks(system, block_rewrite)) or system) if block_rewrite else system
|
||||
|
||||
# Build a typed AnthropicMessagesRequest for the adapter
|
||||
request_data: Final[AnthropicMessagesRequest] = {
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ path used for OpenAI and Azure models.
|
|||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable, Mapping
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from itertools import groupby
|
||||
from typing import Any, Final, cast
|
||||
|
||||
|
|
@ -127,17 +127,93 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
return {"type": "input_file", "file_url": url}
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _content_blocks(content: object) -> tuple[Mapping[str, object], ...]:
|
||||
if not isinstance(content, list):
|
||||
return ()
|
||||
return tuple(
|
||||
cast(Mapping[str, object], block) # cast-ok: untrusted client payload
|
||||
for block in cast(Sequence[object], content) # cast-ok: untyped client content list
|
||||
if isinstance(block, dict)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _tool_result_image_parts(
|
||||
cls, blocks: Sequence[Mapping[str, object]]
|
||||
) -> tuple[dict[str, object], ...]: # mutable-ok: json parts
|
||||
"""Each image block as an input_image part, keeping a breakpoint the image carries."""
|
||||
sources: Final = tuple(
|
||||
(cls._translate_anthropic_image_source_to_url(block.get("source")), block.get("prompt_cache_breakpoint"))
|
||||
for block in blocks
|
||||
if block.get("type") == "image"
|
||||
)
|
||||
return tuple(
|
||||
with_prompt_cache_breakpoint({"type": "input_image", "image_url": url}, breakpoint)
|
||||
for url, breakpoint in sources
|
||||
if url
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _tool_result_images_by_breakpoint(
|
||||
cls, block: Mapping[str, object]
|
||||
) -> tuple[object, tuple[dict[str, object], ...], tuple[dict[str, object], ...]]: # mutable-ok: json parts
|
||||
"""The breakpoint covering a tool_result, its images at or before that breakpoint, and its images after it.
|
||||
|
||||
A mark on the tool_result covers all its images, and a mark on a block inside it covers the images up to that
|
||||
block, the images Anthropic caches for the mark. The result's text and documents join into one output, which
|
||||
the mark covers whole.
|
||||
"""
|
||||
inner: Final = cls._content_blocks(block.get("content"))
|
||||
own_breakpoint: Final = block.get("prompt_cache_breakpoint")
|
||||
if own_breakpoint is not None:
|
||||
return own_breakpoint, cls._tool_result_image_parts(inner), ()
|
||||
last_marked: Final = next(
|
||||
(
|
||||
position
|
||||
for position in reversed(range(len(inner)))
|
||||
if inner[position].get("prompt_cache_breakpoint") is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
if last_marked is None:
|
||||
return None, (), cls._tool_result_image_parts(inner)
|
||||
return (
|
||||
inner[last_marked].get("prompt_cache_breakpoint"),
|
||||
cls._tool_result_image_parts(inner[: last_marked + 1]),
|
||||
cls._tool_result_image_parts(inner[last_marked + 1 :]),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _tool_result_images_message(
|
||||
image_parts: Sequence[Mapping[str, object]],
|
||||
prompt_cache_breakpoint: object,
|
||||
) -> dict[str, object]: # mutable-ok: API message payload
|
||||
"""Tool result images as one user message, the tool_result's breakpoint on the last image.
|
||||
|
||||
OpenAI caches only up to a breakpoint, so a marked result's images go right after its output and ahead of any
|
||||
later output, the images Anthropic caches for that mark.
|
||||
"""
|
||||
*earlier_images, last_image = image_parts
|
||||
return {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": TOOL_RESULT_IMAGE_BOUNDARY},
|
||||
*earlier_images,
|
||||
with_prompt_cache_breakpoint(last_image, prompt_cache_breakpoint),
|
||||
],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _tool_result_output_value(
|
||||
output_text: str,
|
||||
file_parts: tuple[dict[str, str], ...], # mutable-ok: json content parts
|
||||
tool_result: Mapping[str, object],
|
||||
prompt_cache_breakpoint: object,
|
||||
) -> str | list[dict[str, str]]: # mutable-ok: API message payload
|
||||
"""Plain string output, or a part list when document file parts or a cache breakpoint are present.
|
||||
|
||||
OpenAI reads a breakpoint only on a part, so a marked result ends on a part that carries it.
|
||||
"""
|
||||
prompt_cache_breakpoint: Final = tool_result.get("prompt_cache_breakpoint")
|
||||
text_parts: Final = [{"type": "input_text", "text": output_text}] if output_text else []
|
||||
parts: Final = [*text_parts, *file_parts]
|
||||
if not parts or (not file_parts and prompt_cache_breakpoint is None):
|
||||
|
|
@ -292,6 +368,11 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
elif btype == "tool_result":
|
||||
tool_use_id = block.get("tool_use_id", "")
|
||||
inner = block.get("content")
|
||||
tool_breakpoint, covered_image_parts, later_image_parts = (
|
||||
self._tool_result_images_by_breakpoint(
|
||||
cast(Mapping[str, object], block) # cast-ok: TypedDict upcast
|
||||
)
|
||||
)
|
||||
document_candidates = (
|
||||
tuple(
|
||||
self._translate_anthropic_document_block_to_file_part(c)
|
||||
|
|
@ -311,43 +392,37 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
c.get("text", "") for c in inner if isinstance(c, dict) and c.get("type") == "text"
|
||||
]
|
||||
output_text = "\n".join(parts)
|
||||
image_candidates = tuple(
|
||||
self._translate_anthropic_image_source_to_url(c.get("source"))
|
||||
for c in inner
|
||||
if isinstance(c, dict) and c.get("type") == "image"
|
||||
)
|
||||
image_urls = tuple(url for url in image_candidates if url)
|
||||
if image_urls:
|
||||
if covered_image_parts or later_image_parts:
|
||||
output_text = (
|
||||
f"{output_text}\n{TOOL_RESULT_IMAGE_PLACEHOLDER}"
|
||||
if output_text
|
||||
else TOOL_RESULT_IMAGE_PLACEHOLDER
|
||||
)
|
||||
tool_image_parts.extend(
|
||||
{"type": "input_image", "image_url": url} for url in image_urls
|
||||
)
|
||||
else:
|
||||
output_text = str(inner)
|
||||
images_inside_breakpoint = (
|
||||
(*tool_image_parts, *covered_image_parts) if tool_breakpoint is not None else ()
|
||||
)
|
||||
# tool_result is a top-level item, not inside the message
|
||||
input_items.append(
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": tool_use_id,
|
||||
"output": self._tool_result_output_value(output_text, tool_file_parts, block),
|
||||
"output": self._tool_result_output_value(
|
||||
output_text,
|
||||
tool_file_parts,
|
||||
None if images_inside_breakpoint else tool_breakpoint,
|
||||
),
|
||||
}
|
||||
)
|
||||
if images_inside_breakpoint:
|
||||
input_items.append(
|
||||
self._tool_result_images_message(images_inside_breakpoint, tool_breakpoint)
|
||||
)
|
||||
tool_image_parts.clear()
|
||||
tool_image_parts.extend(later_image_parts)
|
||||
if tool_image_parts:
|
||||
boundary_part = {
|
||||
"type": "input_text",
|
||||
"text": TOOL_RESULT_IMAGE_BOUNDARY,
|
||||
}
|
||||
input_items.append(
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [boundary_part, *tool_image_parts],
|
||||
}
|
||||
)
|
||||
input_items.append(self._tool_result_images_message(tool_image_parts, None))
|
||||
if user_parts:
|
||||
input_items.append(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -485,6 +485,42 @@ def test_anthropic_sdk_cache_control_on_a_tool_result_marks_its_function_call_ou
|
|||
], "OpenAI caches up to a breakpoint on a function_call_output part (prompt caching guide, read 2026-10-12)"
|
||||
|
||||
|
||||
def test_anthropic_sdk_cache_control_on_a_tool_result_with_a_screenshot_marks_the_screenshot(bridge: _Bridge) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
screenshot_result: Final = {
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "call_1",
|
||||
"content": [
|
||||
{"type": "text", "text": "captured"},
|
||||
{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "Zm9v"}},
|
||||
],
|
||||
"cache_control": _EPHEMERAL,
|
||||
}
|
||||
with bridge.gateway.scenario() as scenario, _anthropic(bridge) as client:
|
||||
model: Final = scenario.model(model=pcb.MODEL, api_base=bridge.api_base, **_OPTED_IN)
|
||||
message: Final = _anthropic_message(
|
||||
client,
|
||||
model,
|
||||
[
|
||||
{"role": "user", "content": pcb.prompt(marker)},
|
||||
{"role": "assistant", "content": [{"type": "tool_use", "id": "call_1", "name": "shot", "input": {}}]},
|
||||
{"role": "user", "content": [screenshot_result]},
|
||||
],
|
||||
)
|
||||
_anthropic_text(message, marker)
|
||||
request: Final = pcb.posted(bridge.wire, marker)
|
||||
_wire_body(request)
|
||||
items: Final = pcb.input_items(request)
|
||||
output_index: Final = next(index for index, item in enumerate(items) if item.get("call_id") == "call_1")
|
||||
*_, screenshot = pcb.content_of(items[output_index + 1 :], "user")
|
||||
assert screenshot == {
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,Zm9v",
|
||||
"prompt_cache_breakpoint": pcb.EXPLICIT,
|
||||
}, "OpenAI caches only the prefix up to the breakpoint, so the screenshot must sit inside it"
|
||||
assert request.body.count(b"prompt_cache_breakpoint") == 1, request.body
|
||||
|
||||
|
||||
def test_anthropic_sdk_cache_control_on_an_assistant_turn_is_dropped(bridge: _Bridge) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
messages: Final[list[anthropic.types.MessageParam]] = [
|
||||
|
|
|
|||
|
|
@ -11,6 +11,10 @@ import respx
|
|||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../..")))
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
TOOL_RESULT_IMAGE_BOUNDARY,
|
||||
TOOL_RESULT_IMAGE_PLACEHOLDER,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.responses_adapters.handler import (
|
||||
LiteLLMMessagesToResponsesAPIHandler,
|
||||
_build_responses_kwargs,
|
||||
|
|
@ -255,6 +259,110 @@ def test_client_cache_control_reaches_gpt_5_6_as_a_prompt_cache_breakpoint_on_th
|
|||
]
|
||||
|
||||
|
||||
def test_client_cache_control_on_a_screenshot_inside_a_tool_result_reaches_gpt_5_6_on_that_screenshot() -> None:
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
model="openai/gpt-5.6",
|
||||
extra_kwargs=OPENAI_API,
|
||||
max_tokens=1024,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_shot",
|
||||
"content": [
|
||||
{
|
||||
"type": "image",
|
||||
"source": {"type": "base64", "media_type": "image/png", "data": "c2hvdA=="},
|
||||
"cache_control": EPHEMERAL,
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert responses_kwargs["input"] == [
|
||||
{"type": "function_call_output", "call_id": "toolu_shot", "output": TOOL_RESULT_IMAGE_PLACEHOLDER},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": TOOL_RESULT_IMAGE_BOUNDARY},
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,c2hvdA==",
|
||||
"prompt_cache_breakpoint": {"mode": "explicit"},
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
RAW_BREAKPOINT: Final = {"mode": "explicit"}
|
||||
|
||||
|
||||
RAW_LOG_OUTPUT: Final = [{"type": "input_text", "text": "build ok", "prompt_cache_breakpoint": RAW_BREAKPOINT}]
|
||||
RAW_QUESTION: Final = {"type": "input_text", "text": "q", "prompt_cache_breakpoint": RAW_BREAKPOINT}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "extra_kwargs", "log_output", "question"),
|
||||
[
|
||||
pytest.param("openai/gpt-5.5", OPENAI_API, RAW_LOG_OUTPUT, RAW_QUESTION, id="sent-as-is"),
|
||||
pytest.param(
|
||||
"openai/gpt-5.5",
|
||||
{**OPENAI_API, "drop_params": True},
|
||||
"build ok",
|
||||
{"type": "input_text", "text": "q"},
|
||||
id="dropped-under-drop-params",
|
||||
),
|
||||
pytest.param(
|
||||
"openai/gpt-5.6",
|
||||
{**OTHER_HOST, "drop_params": True},
|
||||
RAW_LOG_OUTPUT,
|
||||
RAW_QUESTION,
|
||||
id="kept-under-drop-params-for-a-model-that-takes-them",
|
||||
),
|
||||
pytest.param(
|
||||
"openai/screenshots-prod",
|
||||
{**OTHER_HOST, "drop_params": True, "base_model": "gpt-5.6"},
|
||||
RAW_LOG_OUTPUT,
|
||||
RAW_QUESTION,
|
||||
id="kept-under-drop-params-for-a-base-model-that-takes-them",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_raw_prompt_cache_breakpoints_reach_a_deployment_unless_drop_params_drops_them_for_a_model_that_takes_none(
|
||||
model: str, extra_kwargs: dict[str, object], log_output: object, question: dict[str, object]
|
||||
) -> None:
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
model=model,
|
||||
extra_kwargs=extra_kwargs,
|
||||
max_tokens=1024,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "q", "prompt_cache_breakpoint": RAW_BREAKPOINT},
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_log",
|
||||
"content": [{"type": "text", "text": "build ok", "prompt_cache_breakpoint": RAW_BREAKPOINT}],
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert responses_kwargs["input"] == [
|
||||
{"type": "function_call_output", "call_id": "toolu_log", "output": log_output},
|
||||
{"type": "message", "role": "user", "content": [question]},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "extra_kwargs"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ Tests for LiteLLMAnthropicToResponsesAPIAdapter
|
|||
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any, Dict, Final, List
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -623,9 +623,7 @@ class TestTranslateMessagesToResponsesInput:
|
|||
}
|
||||
]
|
||||
result = _translate_messages(messages)
|
||||
assert result == [
|
||||
{"type": "reasoning", "summary": [{"type": "summary_text", "text": "Private reasoning."}]}
|
||||
]
|
||||
assert result == [{"type": "reasoning", "summary": [{"type": "summary_text", "text": "Private reasoning."}]}]
|
||||
|
||||
def test_consecutive_thinking_blocks_become_one_reasoning_item(self):
|
||||
"""Summary parts of one upstream reasoning item are regrouped into that item."""
|
||||
|
|
@ -2189,6 +2187,184 @@ class TestPromptCacheBreakpointToResponses:
|
|||
"output": [{"type": "input_text", "text": "r", "prompt_cache_breakpoint": self.EXPLICIT}],
|
||||
}
|
||||
|
||||
def test_a_marked_tool_result_with_screenshots_moves_its_breakpoint_to_the_last_screenshot(self):
|
||||
items: Final = _ADAPTER.translate_messages_to_responses_input(
|
||||
[
|
||||
{"role": "user", "content": [{"type": "text", "text": "q"}]},
|
||||
{"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01", "name": "t", "input": {}}]},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_01",
|
||||
"content": [
|
||||
{"type": "text", "text": "r"},
|
||||
{
|
||||
"type": "image",
|
||||
"source": {"type": "base64", "media_type": "image/png", "data": "Zmlyc3Q="},
|
||||
},
|
||||
{
|
||||
"type": "image",
|
||||
"source": {"type": "base64", "media_type": "image/png", "data": "bGFzdA=="},
|
||||
},
|
||||
],
|
||||
"prompt_cache_breakpoint": self.EXPLICIT,
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
)
|
||||
*_, tool_output, screenshots = items
|
||||
assert tool_output == {
|
||||
"type": "function_call_output",
|
||||
"call_id": "toolu_01",
|
||||
"output": f"r\n{TOOL_RESULT_IMAGE_PLACEHOLDER}",
|
||||
}
|
||||
*_, first_screenshot, last_screenshot = screenshots["content"]
|
||||
assert first_screenshot == {"type": "input_image", "image_url": "data:image/png;base64,Zmlyc3Q="}
|
||||
assert last_screenshot == {
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,bGFzdA==",
|
||||
"prompt_cache_breakpoint": self.EXPLICIT,
|
||||
}
|
||||
|
||||
SCREENSHOT: Final = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "c2hvdA=="}}
|
||||
MARKED_SCREENSHOT_PART: Final = {
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,c2hvdA==",
|
||||
"prompt_cache_breakpoint": EXPLICIT,
|
||||
}
|
||||
LOG_OUTPUT: Final = {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok"}
|
||||
|
||||
def test_an_unmarked_screenshot_before_a_marked_tool_result_lands_inside_that_breakpoint(self):
|
||||
items: Final = _ADAPTER.translate_messages_to_responses_input(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "tool_result", "tool_use_id": "toolu_shot", "content": [self.SCREENSHOT]},
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_log",
|
||||
"content": "build ok",
|
||||
"prompt_cache_breakpoint": self.EXPLICIT,
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
_, log_output, screenshots = items
|
||||
assert log_output == self.LOG_OUTPUT
|
||||
assert screenshots["content"][-1] == self.MARKED_SCREENSHOT_PART
|
||||
|
||||
@pytest.mark.parametrize("mark_on_the_screenshot", [False, True], ids=["mark-on-tool-result", "mark-on-screenshot"])
|
||||
def test_a_tool_result_after_a_marked_screenshot_stays_after_its_breakpoint(self, mark_on_the_screenshot: bool):
|
||||
mark: Final = {"prompt_cache_breakpoint": self.EXPLICIT}
|
||||
marked_result: Final = (
|
||||
{"type": "tool_result", "tool_use_id": "toolu_shot", "content": [{**self.SCREENSHOT, **mark}]}
|
||||
if mark_on_the_screenshot
|
||||
else {"type": "tool_result", "tool_use_id": "toolu_shot", "content": [self.SCREENSHOT], **mark}
|
||||
)
|
||||
items: Final = _ADAPTER.translate_messages_to_responses_input(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
marked_result,
|
||||
{"type": "tool_result", "tool_use_id": "toolu_log", "content": "build ok"},
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
_, screenshots, log_output = items
|
||||
assert screenshots["content"][-1] == self.MARKED_SCREENSHOT_PART
|
||||
assert log_output == self.LOG_OUTPUT
|
||||
|
||||
def test_a_breakpoint_on_text_inside_a_tool_result_marks_its_output(self):
|
||||
items: Final = _ADAPTER.translate_messages_to_responses_input(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_log",
|
||||
"content": [{"type": "text", "text": "build ok", "prompt_cache_breakpoint": self.EXPLICIT}],
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
assert items == [
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "toolu_log",
|
||||
"output": [{"type": "input_text", "text": "build ok", "prompt_cache_breakpoint": self.EXPLICIT}],
|
||||
}
|
||||
]
|
||||
|
||||
SCREENSHOT_PART: Final = {"type": "input_image", "image_url": "data:image/png;base64,c2hvdA=="}
|
||||
MARKED_LOG_TEXT: Final = {"type": "text", "text": "build ok", "prompt_cache_breakpoint": EXPLICIT}
|
||||
LOG_TEXT_WITH_SCREENSHOT: Final = f"build ok\n{TOOL_RESULT_IMAGE_PLACEHOLDER}"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("content", "output", "screenshot"),
|
||||
[
|
||||
pytest.param(
|
||||
[MARKED_LOG_TEXT, SCREENSHOT],
|
||||
[{"type": "input_text", "text": LOG_TEXT_WITH_SCREENSHOT, "prompt_cache_breakpoint": EXPLICIT}],
|
||||
SCREENSHOT_PART,
|
||||
id="screenshot-after-the-marked-text",
|
||||
),
|
||||
pytest.param(
|
||||
[SCREENSHOT, MARKED_LOG_TEXT],
|
||||
LOG_TEXT_WITH_SCREENSHOT,
|
||||
MARKED_SCREENSHOT_PART,
|
||||
id="screenshot-before-the-marked-text",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_breakpoint_on_text_inside_a_tool_result_covers_only_the_screenshots_before_that_text(
|
||||
self, content: list[dict[str, object]], output: object, screenshot: dict[str, object]
|
||||
):
|
||||
items: Final = _ADAPTER.translate_messages_to_responses_input(
|
||||
[{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_shot", "content": content}]}]
|
||||
)
|
||||
assert items == [
|
||||
{"type": "function_call_output", "call_id": "toolu_shot", "output": output},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": TOOL_RESULT_IMAGE_BOUNDARY}, screenshot],
|
||||
},
|
||||
]
|
||||
|
||||
def test_a_later_tool_output_stays_after_the_last_marked_screenshot_of_an_earlier_result(self):
|
||||
marked_screenshot: Final = {**self.SCREENSHOT, "prompt_cache_breakpoint": self.EXPLICIT}
|
||||
items: Final = _ADAPTER.translate_messages_to_responses_input(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_shot",
|
||||
"content": [marked_screenshot, marked_screenshot],
|
||||
},
|
||||
{"type": "tool_result", "tool_use_id": "toolu_log", "content": "build ok"},
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
_, screenshots, log_output = items
|
||||
assert screenshots["content"] == [
|
||||
{"type": "input_text", "text": TOOL_RESULT_IMAGE_BOUNDARY},
|
||||
self.MARKED_SCREENSHOT_PART,
|
||||
self.MARKED_SCREENSHOT_PART,
|
||||
]
|
||||
assert log_output == self.LOG_OUTPUT
|
||||
|
||||
def test_prompt_cache_options_forwarded_to_responses_kwargs(self):
|
||||
from litellm.llms.anthropic.pass_through.responses_adapters.handler import (
|
||||
_build_responses_kwargs,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue