mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-17 04:34:25 +00:00
fix(anthropic): keep trailing user messages with None or empty list content (#41397)
* fix(anthropic): keep user messages with None or empty list content so requests do not end on an assistant turn Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): avoid mutable empty-list construction in sanitizer Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): avoid rebinding sanitizer input Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): keep a trailing user turn whose content key was dropped litellm.completion strips None fields from every message before the provider transform, so a user message sent with content null reaches the Anthropic converter with no content key and raised KeyError. Read the content with get so that turn gets the same placeholder as an empty list. * fix(anthropic): only fill user turns with empty content and keep their cache_control * fix(anthropic): fill each empty user message in place like an empty string * fix(anthropic): add the placeholder only for an empty turn or a cache breakpoint, in linear time * test(anthropic): cover the tool-result guard with cache_control and widen the linear-time bound * test(anthropic): cover empty user turns through the proxy on chat, responses and messages * test(anthropic): type the empty user turn tests and move the linear-time check to a benchmark --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
f281f77bc0
commit
4e49394ade
7 changed files with 506 additions and 35 deletions
|
|
@ -2582,6 +2582,48 @@ def _with_native_redacted_data(
|
|||
_AnthropicMessageList: TypeAlias = list[AllAnthropicPassThroughMessageValues]
|
||||
|
||||
|
||||
def _client_tool_result(message: AllMessageValues, replaced_server_tool_answers: frozenset[str]) -> bool:
|
||||
return (
|
||||
message_field(message, "role") in ("tool", "function")
|
||||
and message_field(message, "tool_call_id") not in replaced_server_tool_answers
|
||||
)
|
||||
|
||||
|
||||
def _tool_result_later_in_turn(
|
||||
messages: Sequence[AllMessageValues], replaced_server_tool_answers: frozenset[str]
|
||||
) -> tuple[bool, ...]:
|
||||
from_the_end: Final = itertools.accumulate(
|
||||
reversed(messages),
|
||||
lambda later, message: (
|
||||
message_field(message, "role") in ("user", "tool", "function")
|
||||
and (later or _client_tool_result(message, replaced_server_tool_answers))
|
||||
),
|
||||
initial=False,
|
||||
)
|
||||
return tuple(from_the_end)[::-1][1:]
|
||||
|
||||
|
||||
def _empty_text_placeholder(message: AllMessageValues) -> AnthropicMessagesTextParam:
|
||||
placeholder: Final = AnthropicMessagesTextParam(type="text", text=_EMPTY_TEXT_PLACEHOLDER)
|
||||
add_cache_control_to_content(anthropic_content_element=placeholder, original_content_element=message)
|
||||
return placeholder
|
||||
|
||||
|
||||
def _cache_breakpoint_placeholder(
|
||||
message: AllMessageValues, tool_result_follows: bool
|
||||
) -> tuple[AnthropicMessagesTextParam, ...]:
|
||||
if tool_result_follows or not message_field(message, "cache_control"):
|
||||
return ()
|
||||
return (_empty_text_placeholder(message),)
|
||||
|
||||
|
||||
def _empty_user_turn_placeholder(span: Sequence[AllMessageValues]) -> AnthropicMessagesTextParam | None:
|
||||
empty_user_message: Final = next(
|
||||
(message for message in reversed(span) if message["role"] == "user" and not message.get("content")), None
|
||||
)
|
||||
return None if empty_user_message is None else _empty_text_placeholder(empty_user_message)
|
||||
|
||||
|
||||
def anthropic_messages_pt(
|
||||
messages: list[AllMessageValues],
|
||||
model: str,
|
||||
|
|
@ -2636,6 +2678,7 @@ def anthropic_messages_pt(
|
|||
is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False
|
||||
force_base64 = is_bedrock_invoke or is_vertex_ai
|
||||
|
||||
tool_result_later_in_turn: Final = _tool_result_later_in_turn(messages, replaced_server_tool_answers)
|
||||
msg_i = 0
|
||||
while msg_i < len(messages):
|
||||
user_content: list[AnthropicMessagesUserMessageValues] = []
|
||||
|
|
@ -2648,8 +2691,12 @@ def anthropic_messages_pt(
|
|||
ChatCompletionToolMessage | ChatCompletionUserMessage | ChatCompletionFunctionMessage
|
||||
) = messages[msg_i]
|
||||
if user_message_types_block["role"] == "user":
|
||||
if isinstance(user_message_types_block["content"], list):
|
||||
for m in user_message_types_block["content"]:
|
||||
if not (user_block_content := user_message_types_block.get("content")):
|
||||
user_content.extend(
|
||||
_cache_breakpoint_placeholder(user_message_types_block, tool_result_later_in_turn[msg_i])
|
||||
)
|
||||
elif isinstance(user_block_content, list):
|
||||
for m in user_block_content:
|
||||
if m.get("type", "") == "image_url":
|
||||
m = cast(ChatCompletionImageObject, m)
|
||||
format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None
|
||||
|
|
@ -2720,10 +2767,10 @@ def anthropic_messages_pt(
|
|||
_file_content_element,
|
||||
)
|
||||
)
|
||||
elif isinstance(user_message_types_block["content"], str):
|
||||
elif isinstance(user_block_content, str):
|
||||
_anthropic_content_text_element: AnthropicMessagesTextParam = {
|
||||
"type": "text",
|
||||
"text": user_message_types_block["content"],
|
||||
"text": user_block_content,
|
||||
}
|
||||
_content_element = add_cache_control_to_content(
|
||||
anthropic_content_element=_anthropic_content_text_element,
|
||||
|
|
@ -2747,6 +2794,8 @@ def anthropic_messages_pt(
|
|||
|
||||
if user_content:
|
||||
new_messages.append({"role": "user", "content": user_content})
|
||||
elif (placeholder := _empty_user_turn_placeholder(messages[init_msg_i:msg_i])) is not None:
|
||||
new_messages.append({"role": "user", "content": [placeholder]})
|
||||
|
||||
# Track unique tool IDs in this merge block to avoid duplication
|
||||
unique_tool_ids: set[str] = set()
|
||||
|
|
|
|||
|
|
@ -414,10 +414,11 @@ def _assistant_renders(message: object, answered: frozenset[str]) -> bool:
|
|||
def _renders(message: object, answered: frozenset[str]) -> bool:
|
||||
"""Whether ``anthropic_messages_pt`` puts a block on the wire for this message.
|
||||
|
||||
A tool message always becomes a ``tool_result`` and a user message with string
|
||||
content always becomes a text block (empty text gets a placeholder). A user list
|
||||
renders only through parts of a type the converter emits; ``None``, an empty list,
|
||||
and a list of other parts vanish. Assistant messages follow ``_assistant_renders``.
|
||||
A tool message always becomes a ``tool_result``, and a user message with string or
|
||||
missing content (``None``, an empty list) always puts its turn on the wire, an empty
|
||||
turn getting a placeholder. A user list renders only through parts of a type the
|
||||
converter emits, so a list of only other parts vanishes. Assistant messages follow
|
||||
``_assistant_renders``.
|
||||
"""
|
||||
role: Final = message_field(message, "role")
|
||||
if role in _TOOL_ROLES:
|
||||
|
|
@ -425,8 +426,10 @@ def _renders(message: object, answered: frozenset[str]) -> bool:
|
|||
if role == "assistant":
|
||||
return _assistant_renders(message, answered)
|
||||
content: Final = message_field(message, "content")
|
||||
return isinstance(content, str) or any(
|
||||
message_field(part, "type") in _RENDERED_PART_TYPES for part in parts_of(content)
|
||||
return (
|
||||
isinstance(content, str)
|
||||
or not content
|
||||
or any(message_field(part, "type") in _RENDERED_PART_TYPES for part in parts_of(content))
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
21
tests/benchmarks/test_prompt_template_benchmarks.py
Normal file
21
tests/benchmarks/test_prompt_template_benchmarks.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
|
||||
|
||||
LONG_RUN_OF_EMPTY_CACHED_USER_MESSAGES: Final[tuple[Mapping[str, object], ...]] = (
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "hello"},
|
||||
*({"role": "user", "content": [], "cache_control": {"type": "ephemeral"}} for _ in range(1_000)),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_anthropic_messages_pt_long_run_of_empty_cached_user_messages() -> None:
|
||||
result: Final = anthropic_messages_pt(
|
||||
messages=list(LONG_RUN_OF_EMPTY_CACHED_USER_MESSAGES), model="claude-opus-5", llm_provider="vertex_ai"
|
||||
)
|
||||
|
||||
assert len(result[-1]["content"]) == 1_000
|
||||
|
|
@ -0,0 +1,216 @@
|
|||
import json
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, gateway_from_environment, object_value
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from integration.providers._cache_control_marks_support import (
|
||||
anthropic_deployment,
|
||||
anthropic_peer,
|
||||
final_text,
|
||||
new_marker,
|
||||
)
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_MODEL: Final = "empty-user-turn"
|
||||
_PLACEHOLDER: Final = "[System: Empty message content sanitised to satisfy protocol]"
|
||||
_REPLY: Final = "Noted."
|
||||
_EMPTY_ROWS: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType(
|
||||
{
|
||||
"list": MappingProxyType({"role": "user", "content": []}),
|
||||
"null": MappingProxyType({"role": "user", "content": None}),
|
||||
"missing": MappingProxyType({"role": "user"}),
|
||||
"empty-string": MappingProxyType({"role": "user", "content": ""}),
|
||||
}
|
||||
)
|
||||
_CHAT_EMPTY_KINDS: Final = ("list", "null", "missing")
|
||||
|
||||
Turn = tuple[str, tuple[str, ...]]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Rig:
|
||||
gateway: Gateway
|
||||
provider: Wire
|
||||
|
||||
|
||||
def _empty_row(kind: str) -> dict[str, JsonValue]:
|
||||
return dict(_EMPTY_ROWS[kind])
|
||||
|
||||
|
||||
def _guardrail_peer(request: Request) -> Reply:
|
||||
kind: Final = request.target.split("/")[1]
|
||||
rows: Final = _JSON_OBJECT.validate_json(request.body)["structured_messages"]
|
||||
assert isinstance(rows, list), rows
|
||||
verdict: Final = {"action": "GUARDRAIL_INTERVENED", "structured_messages": [*rows, _empty_row(kind)]}
|
||||
return Reply(body=json.dumps(verdict).encode())
|
||||
|
||||
|
||||
def _guardrail(kind: str, api_base: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"guardrail_name": f"append-{kind}",
|
||||
"litellm_params": {
|
||||
"guardrail": "generic_guardrail_api",
|
||||
"mode": "pre_call",
|
||||
"default_on": False,
|
||||
"api_base": f"{api_base}/{kind}",
|
||||
"api_key": "synthetic-guardrail-key",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _config(directory: Path, provider_url: str, guardrail_url: str) -> Path:
|
||||
stock: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
config: Final[dict[str, JsonValue]] = {
|
||||
**stock,
|
||||
"model_list": [anthropic_deployment(_MODEL, provider_url)],
|
||||
"router_settings": {**object_value(stock["router_settings"]), "num_retries": 0},
|
||||
"guardrails": [_guardrail(kind, guardrail_url) for kind in _EMPTY_ROWS],
|
||||
}
|
||||
path: Final = directory / "empty-user-turn.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
|
||||
directory: Final = tmp_path_factory.mktemp("empty-user-turn")
|
||||
with (
|
||||
wire_server(anthropic_peer) as provider,
|
||||
wire_server(_guardrail_peer) as guardrail,
|
||||
gateway_from_environment() as gateway,
|
||||
):
|
||||
config: Final = _config(directory, provider.url, guardrail.url)
|
||||
with owned_proxy(gateway, directory, {}, config=config, workers=2) as owned:
|
||||
yield _Rig(owned, provider)
|
||||
|
||||
|
||||
def _block(block: JsonValue) -> str:
|
||||
part: Final = object_value(block)
|
||||
return str(part["text"]) if part.get("type") == "text" else str(part.get("type"))
|
||||
|
||||
|
||||
def _turn(message: JsonValue) -> Turn:
|
||||
row: Final = object_value(message)
|
||||
content: Final = row["content"]
|
||||
if isinstance(content, str):
|
||||
return (str(row["role"]), (content,))
|
||||
assert isinstance(content, list), row
|
||||
return (str(row["role"]), tuple(_block(block) for block in content))
|
||||
|
||||
|
||||
def _send(rig: _Rig, path: str, body: Mapping[str, JsonValue]) -> None:
|
||||
response: Final = rig.gateway.client.post(
|
||||
path, json=dict(body), headers={"Authorization": f"Bearer {rig.gateway.key}"}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
|
||||
def _provider_turns(rig: _Rig, marker: str) -> tuple[Turn, ...]:
|
||||
needle: Final = marker.encode()
|
||||
requests: Final = [request for request in rig.provider.drain() if needle in request.body]
|
||||
assert len(requests) == 1, requests
|
||||
messages: Final = _JSON_OBJECT.validate_json(requests[0].body)["messages"]
|
||||
assert isinstance(messages, list), messages
|
||||
return tuple(_turn(message) for message in messages)
|
||||
|
||||
|
||||
def _ask_row(ask: str) -> dict[str, JsonValue]:
|
||||
return {"role": "user", "content": ask}
|
||||
|
||||
|
||||
def _chat(messages: Sequence[JsonValue]) -> dict[str, JsonValue]:
|
||||
return {"model": _MODEL, "max_tokens": 64, "messages": list(messages)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", _CHAT_EMPTY_KINDS)
|
||||
def test_chat_trailing_empty_user_message_keeps_a_user_turn(rig: _Rig, kind: str) -> None:
|
||||
marker: Final = new_marker()
|
||||
ask: Final = final_text(marker)
|
||||
_send(
|
||||
rig,
|
||||
"/v1/chat/completions",
|
||||
_chat([_ask_row(ask), {"role": "assistant", "content": _REPLY}, _empty_row(kind)]),
|
||||
)
|
||||
assert _provider_turns(rig, marker) == (("user", (ask,)), ("assistant", (_REPLY,)), ("user", (_PLACEHOLDER,)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", _CHAT_EMPTY_KINDS)
|
||||
def test_chat_empty_user_message_beside_text_sends_only_the_text(rig: _Rig, kind: str) -> None:
|
||||
marker: Final = new_marker()
|
||||
ask: Final = final_text(marker)
|
||||
_send(rig, "/v1/chat/completions", _chat([_ask_row(ask), _empty_row(kind)]))
|
||||
assert _provider_turns(rig, marker) == (("user", (ask,)),)
|
||||
|
||||
|
||||
def _system_then_empty_turns(rig: _Rig, kind: str) -> tuple[Turn, ...]:
|
||||
marker: Final = new_marker()
|
||||
ask: Final = final_text(marker)
|
||||
_send(
|
||||
rig,
|
||||
"/v1/chat/completions",
|
||||
_chat(
|
||||
[
|
||||
_ask_row(ask),
|
||||
{"role": "assistant", "content": _REPLY},
|
||||
{"role": "system", "content": "Answer in French."},
|
||||
_empty_row(kind),
|
||||
]
|
||||
),
|
||||
)
|
||||
turns: Final = _provider_turns(rig, marker)
|
||||
assert turns[0] == ("user", (ask,)), turns
|
||||
return turns[1:]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", _CHAT_EMPTY_KINDS)
|
||||
def test_chat_system_message_before_an_empty_user_message_is_placed_like_an_empty_string(rig: _Rig, kind: str) -> None:
|
||||
assert _system_then_empty_turns(rig, kind) == _system_then_empty_turns(rig, "empty-string")
|
||||
|
||||
|
||||
def test_responses_trailing_empty_input_message_keeps_a_user_turn(rig: _Rig) -> None:
|
||||
marker: Final = new_marker()
|
||||
ask: Final = final_text(marker)
|
||||
body: Final[dict[str, JsonValue]] = {
|
||||
"model": _MODEL,
|
||||
"max_output_tokens": 64,
|
||||
"input": [
|
||||
_ask_row(ask),
|
||||
{"role": "assistant", "content": _REPLY},
|
||||
{"role": "user", "content": []},
|
||||
],
|
||||
}
|
||||
_send(rig, "/v1/responses", body)
|
||||
assert _provider_turns(rig, marker) == (("user", (ask,)), ("assistant", (_REPLY,)), ("user", (_PLACEHOLDER,)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", tuple(_EMPTY_ROWS))
|
||||
def test_messages_guardrail_empty_user_row_is_sent_like_an_empty_string(rig: _Rig, kind: str) -> None:
|
||||
marker: Final = new_marker()
|
||||
ask: Final = final_text(marker)
|
||||
body: Final[dict[str, JsonValue]] = {
|
||||
"model": _MODEL,
|
||||
"max_tokens": 64,
|
||||
"messages": [_ask_row(ask)],
|
||||
"guardrails": [f"append-{kind}"],
|
||||
}
|
||||
_send(rig, "/v1/messages", body)
|
||||
assert _provider_turns(rig, marker) == (("user", (ask,)), ("user", (_PLACEHOLDER,)))
|
||||
|
||||
|
||||
def test_a_long_run_of_empty_user_messages_is_sent_as_one_text_turn(rig: _Rig) -> None:
|
||||
marker: Final = new_marker()
|
||||
ask: Final = final_text(marker)
|
||||
_send(
|
||||
rig, "/v1/chat/completions", _chat([*({"role": "user", "content": []} for _ in range(10_000)), _ask_row(ask)])
|
||||
)
|
||||
assert _provider_turns(rig, marker) == (("user", (ask,)),)
|
||||
|
|
@ -4,9 +4,9 @@ import logging
|
|||
import os
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from copy import deepcopy
|
||||
from typing import Final, List
|
||||
from typing import Final, List, TypeAlias
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -16,6 +16,7 @@ import respx
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
BEDROCK_DOCUMENT_PLACEHOLDER_TEXT,
|
||||
_EMPTY_TEXT_PLACEHOLDER,
|
||||
BedrockConverseMessagesProcessor,
|
||||
BedrockImageProcessor,
|
||||
_bedrock_converse_messages_pt,
|
||||
|
|
@ -795,6 +796,138 @@ def test_anthropic_messages_pt_keeps_signed_thinking_block():
|
|||
assert thinking_blocks[0]["thinking"] == "genuine anthropic reasoning"
|
||||
|
||||
|
||||
EMPTY_USER_MESSAGES: Final = pytest.mark.parametrize(
|
||||
"empty_user_message",
|
||||
[{"role": "user", "content": None}, {"role": "user", "content": []}, {"role": "user"}],
|
||||
ids=["null", "empty-list", "content-key-dropped"],
|
||||
)
|
||||
|
||||
Message: TypeAlias = Mapping[str, object]
|
||||
|
||||
|
||||
def _convert(messages: Sequence[Message]) -> Sequence[Message]:
|
||||
return anthropic_messages_pt(messages=list(messages), model="claude-opus-5", llm_provider="vertex_ai")
|
||||
|
||||
|
||||
@EMPTY_USER_MESSAGES
|
||||
def test_anthropic_messages_pt_preserves_trailing_user_message_with_empty_content(empty_user_message: Message) -> None:
|
||||
result: Final = _convert(
|
||||
[
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "hello"},
|
||||
empty_user_message,
|
||||
]
|
||||
)
|
||||
|
||||
assert [message["role"] for message in result] == ["user", "assistant", "user"]
|
||||
assert result[-1]["content"] == [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}]
|
||||
|
||||
|
||||
@EMPTY_USER_MESSAGES
|
||||
@pytest.mark.parametrize(
|
||||
"cache_control", [{}, {"cache_control": {"type": "ephemeral"}}], ids=["no-cache-control", "with-cache-control"]
|
||||
)
|
||||
def test_anthropic_messages_pt_empty_user_message_beside_tool_result_keeps_tool_result_only(
|
||||
empty_user_message: Message, cache_control: Message
|
||||
) -> None:
|
||||
result: Final = _convert(
|
||||
[
|
||||
{"role": "user", "content": "hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "toolu_1", "type": "function", "function": {"name": "f", "arguments": "{}"}}],
|
||||
},
|
||||
{**empty_user_message, **cache_control},
|
||||
{"role": "tool", "tool_call_id": "toolu_1", "content": "ok"},
|
||||
]
|
||||
)
|
||||
|
||||
assert [message["role"] for message in result] == ["user", "assistant", "user"]
|
||||
assert [block["type"] for block in result[-1]["content"]] == ["tool_result"]
|
||||
|
||||
|
||||
@EMPTY_USER_MESSAGES
|
||||
def test_anthropic_messages_pt_empty_user_message_keeps_cache_control_like_an_empty_string(
|
||||
empty_user_message: Message,
|
||||
) -> None:
|
||||
def last_turn(last_user_message: Message) -> Message:
|
||||
return _convert(
|
||||
[
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "hello"},
|
||||
{**last_user_message, "cache_control": {"type": "ephemeral"}},
|
||||
]
|
||||
)[-1]
|
||||
|
||||
assert last_turn(empty_user_message)["content"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
assert last_turn(empty_user_message) == last_turn({"role": "user", "content": ""})
|
||||
|
||||
|
||||
TOOL_CALL_AND_RESULT: Final[tuple[Message, ...]] = (
|
||||
{"role": "user", "content": "hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "toolu_1", "type": "function", "function": {"name": "f", "arguments": "{}"}}],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "toolu_1", "content": "ok"},
|
||||
)
|
||||
|
||||
|
||||
@EMPTY_USER_MESSAGES
|
||||
@pytest.mark.parametrize(
|
||||
"messages_before",
|
||||
[
|
||||
[
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "hello"},
|
||||
{"role": "user", "content": "q"},
|
||||
],
|
||||
[{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}, {"role": "user", "content": []}],
|
||||
TOOL_CALL_AND_RESULT,
|
||||
],
|
||||
ids=["after-user-text", "after-another-empty-message", "after-tool-result"],
|
||||
)
|
||||
def test_anthropic_messages_pt_empty_user_message_beside_other_content_sends_nothing_extra(
|
||||
empty_user_message: Message, messages_before: Sequence[Message]
|
||||
) -> None:
|
||||
assert _convert([*messages_before, empty_user_message]) == _convert(messages_before)
|
||||
|
||||
|
||||
@EMPTY_USER_MESSAGES
|
||||
def test_anthropic_messages_pt_empty_user_message_with_cache_control_after_text_keeps_its_breakpoint(
|
||||
empty_user_message: Message,
|
||||
) -> None:
|
||||
def last_turn(last_user_message: Message) -> Message:
|
||||
return _convert(
|
||||
[
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "hello"},
|
||||
{"role": "user", "content": "long document"},
|
||||
{**last_user_message, "cache_control": {"type": "ephemeral"}},
|
||||
]
|
||||
)[-1]
|
||||
|
||||
assert last_turn(empty_user_message)["content"][-1]["cache_control"] == {"type": "ephemeral"}
|
||||
assert last_turn(empty_user_message) == last_turn({"role": "user", "content": ""})
|
||||
|
||||
|
||||
def test_anthropic_messages_pt_keeps_every_breakpoint_of_a_long_run_of_empty_user_messages() -> None:
|
||||
result: Final = _convert(
|
||||
[
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "hello"},
|
||||
*[{"role": "user", "content": [], "cache_control": {"type": "ephemeral"}}] * 10_000,
|
||||
]
|
||||
)
|
||||
|
||||
assert (
|
||||
result[-1]["content"]
|
||||
== [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER, "cache_control": {"type": "ephemeral"}}] * 10_000
|
||||
)
|
||||
|
||||
|
||||
def test_convert_to_azure_openai_messages():
|
||||
"""Test coverting image_url to azure_openai spec"""
|
||||
|
||||
|
|
@ -4933,7 +5066,9 @@ def test_bedrock_converse_messages_pt_user_message_without_content_adds_no_block
|
|||
|
||||
assert _bedrock_converse_messages_pt(
|
||||
messages=with_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock"
|
||||
) == _bedrock_converse_messages_pt(messages=without_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock")
|
||||
) == _bedrock_converse_messages_pt(
|
||||
messages=without_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -5063,6 +5198,7 @@ def _vcr_outcome_gate(request, vcr):
|
|||
yield
|
||||
record_vcr_outcome(request, vcr)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def isolate_litellm_state():
|
||||
"""
|
||||
|
|
@ -5115,6 +5251,7 @@ def isolate_litellm_state():
|
|||
setattr(litellm, attr, original_value)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
_SCALAR_DEFAULTS = {
|
||||
"num_retries": getattr(litellm, "num_retries", None),
|
||||
"num_retries_per_request": getattr(litellm, "num_retries_per_request", None),
|
||||
|
|
@ -5135,6 +5272,7 @@ _SCALAR_DEFAULTS = {
|
|||
"api_key": getattr(litellm, "api_key", None),
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def setup_and_teardown():
|
||||
"""
|
||||
|
|
@ -5157,6 +5295,7 @@ def setup_and_teardown():
|
|||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_function_call_non_openai_model():
|
||||
try:
|
||||
|
|
@ -5185,6 +5324,7 @@ def test_function_call_non_openai_model():
|
|||
print(e)
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_empty_content():
|
||||
"""
|
||||
|
|
@ -5203,6 +5343,7 @@ def test_empty_content():
|
|||
litellm_call_id=str(uuid.uuid4()),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_thought_signature_removal_for_non_gemini():
|
||||
"""
|
||||
|
|
@ -5251,6 +5392,7 @@ def test_thought_signature_removal_for_non_gemini():
|
|||
assert THOUGHT_SIGNATURE_SEPARATOR not in processed_messages[1]["tool_calls"][0]["id"]
|
||||
assert THOUGHT_SIGNATURE_SEPARATOR not in processed_messages[2]["tool_call_id"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_thought_signature_preserved_for_gemini():
|
||||
"""
|
||||
|
|
@ -5297,6 +5439,7 @@ def test_thought_signature_preserved_for_gemini():
|
|||
assert THOUGHT_SIGNATURE_SEPARATOR in processed_messages[1]["tool_calls"][0]["id"]
|
||||
assert THOUGHT_SIGNATURE_SEPARATOR in processed_messages[2]["tool_call_id"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_thought_signature_removal_with_multiple_tool_calls():
|
||||
"""
|
||||
|
|
@ -5351,6 +5494,7 @@ def test_thought_signature_removal_with_multiple_tool_calls():
|
|||
assert processed_messages[2]["tool_call_id"] == "call_1"
|
||||
assert processed_messages[3]["tool_call_id"] == "call_2"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_messages_without_tool_calls_unchanged():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -29,11 +29,8 @@ def _texts(message: dict) -> list[str]:
|
|||
return [block["text"] for block in message["content"]]
|
||||
|
||||
|
||||
SENDS_NOTHING = pytest.mark.parametrize(
|
||||
"empty_content",
|
||||
[[], None, [{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}]],
|
||||
ids=["empty-list", "none", "unsupported-part-only"],
|
||||
)
|
||||
EMPTY_USER_CONTENT: Final = pytest.mark.parametrize("empty_content", [[], None], ids=["empty-list", "none"])
|
||||
UNSUPPORTED_PART_ONLY: Final = ({"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}},)
|
||||
|
||||
|
||||
def test_split_leading_system_run_keeps_later_system_messages_in_the_conversation():
|
||||
|
|
@ -213,9 +210,9 @@ def test_flagged_placement_of_an_earlier_run_does_not_move_when_later_turns_are_
|
|||
assert _roles(placed_n_plus_one) == ["user", "assistant", "user", "assistant", "user"]
|
||||
|
||||
|
||||
@SENDS_NOTHING
|
||||
def test_flagged_placement_converts_a_run_whose_preceding_user_turn_sends_nothing(empty_content):
|
||||
placed = place_mid_conversation_system(
|
||||
@EMPTY_USER_CONTENT
|
||||
def test_flagged_placement_anchors_a_run_on_a_preceding_empty_user_turn(empty_content: list[object] | None) -> None:
|
||||
placed: Final = place_mid_conversation_system(
|
||||
[
|
||||
{"role": "user", "content": empty_content},
|
||||
{"role": "system", "content": "reminder"},
|
||||
|
|
@ -225,18 +222,48 @@ def test_flagged_placement_converts_a_run_whose_preceding_user_turn_sends_nothin
|
|||
supports_mid_conversation_system=True,
|
||||
)
|
||||
|
||||
assert _roles(placed) == ["user", "system", "assistant", "user"]
|
||||
assert _texts(placed[1]) == ["reminder"]
|
||||
|
||||
|
||||
@EMPTY_USER_CONTENT
|
||||
def test_flagged_placement_anchors_a_run_on_a_following_empty_user_turn(empty_content: list[object] | None) -> None:
|
||||
placed: Final = place_mid_conversation_system(
|
||||
[
|
||||
{"role": "user", "content": "q1"},
|
||||
{"role": "assistant", "content": "a1"},
|
||||
{"role": "system", "content": "reminder"},
|
||||
{"role": "user", "content": empty_content},
|
||||
],
|
||||
supports_mid_conversation_system=True,
|
||||
)
|
||||
|
||||
assert _roles(placed) == ["user", "assistant", "user", "system"]
|
||||
assert _texts(placed[3]) == ["reminder"]
|
||||
|
||||
|
||||
def test_flagged_placement_converts_a_run_whose_preceding_user_turn_sends_nothing() -> None:
|
||||
placed: Final = place_mid_conversation_system(
|
||||
[
|
||||
{"role": "user", "content": list(UNSUPPORTED_PART_ONLY)},
|
||||
{"role": "system", "content": "reminder"},
|
||||
{"role": "assistant", "content": "a1"},
|
||||
{"role": "user", "content": "q2"},
|
||||
],
|
||||
supports_mid_conversation_system=True,
|
||||
)
|
||||
|
||||
assert _roles(placed) == ["user", "user", "assistant", "user"]
|
||||
assert _texts(placed[1]) == [CONVERTED_SYSTEM_NOTE, "reminder"]
|
||||
|
||||
|
||||
@SENDS_NOTHING
|
||||
def test_flagged_placement_converts_a_run_whose_following_user_turn_sends_nothing(empty_content):
|
||||
placed = place_mid_conversation_system(
|
||||
def test_flagged_placement_converts_a_run_whose_following_user_turn_sends_nothing() -> None:
|
||||
placed: Final = place_mid_conversation_system(
|
||||
[
|
||||
{"role": "user", "content": "q1"},
|
||||
{"role": "assistant", "content": "a1"},
|
||||
{"role": "system", "content": "reminder"},
|
||||
{"role": "user", "content": empty_content},
|
||||
{"role": "user", "content": list(UNSUPPORTED_PART_ONLY)},
|
||||
],
|
||||
supports_mid_conversation_system=True,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6763,15 +6763,11 @@ def test_chat_flagged_model_converts_system_with_no_following_user_turn(local_mo
|
|||
assert _texts(result["messages"][2]) == [CONVERTED_SYSTEM_NOTE, REMINDER_TEXT]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"empty_content",
|
||||
[[], None, [{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}]],
|
||||
ids=["empty-list", "none", "unsupported-part-only"],
|
||||
)
|
||||
def test_chat_flagged_model_converts_a_system_behind_a_user_turn_that_sends_nothing(
|
||||
local_model_cost_map, empty_content
|
||||
):
|
||||
messages = [
|
||||
@pytest.mark.parametrize("empty_content", [[], None], ids=["empty-list", "none"])
|
||||
def test_chat_flagged_model_anchors_a_system_on_an_empty_user_turn(
|
||||
local_model_cost_map: None, empty_content: list[object] | None
|
||||
) -> None:
|
||||
messages: Final = [
|
||||
{"role": "system", "content": "You are terse."},
|
||||
{"role": "user", "content": empty_content},
|
||||
{"role": "system", "content": REMINDER_TEXT},
|
||||
|
|
@ -6779,7 +6775,22 @@ def test_chat_flagged_model_converts_a_system_behind_a_user_turn_that_sends_noth
|
|||
{"role": "user", "content": "Second question"},
|
||||
]
|
||||
|
||||
result = _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, messages)
|
||||
result: Final = _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, messages)
|
||||
|
||||
assert [m["role"] for m in result["messages"]] == ["user", "system", "assistant", "user"]
|
||||
assert _texts(result["messages"][1]) == [REMINDER_TEXT]
|
||||
|
||||
|
||||
def test_chat_flagged_model_converts_a_system_behind_a_user_turn_that_sends_nothing(local_model_cost_map: None) -> None:
|
||||
messages: Final = [
|
||||
{"role": "system", "content": "You are terse."},
|
||||
{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}]},
|
||||
{"role": "system", "content": REMINDER_TEXT},
|
||||
{"role": "assistant", "content": "First answer"},
|
||||
{"role": "user", "content": "Second question"},
|
||||
]
|
||||
|
||||
result: Final = _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, messages)
|
||||
|
||||
assert [m["role"] for m in result["messages"]] == ["user", "assistant", "user"]
|
||||
assert _texts(result["messages"][0]) == [CONVERTED_SYSTEM_NOTE, REMINDER_TEXT]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue