mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(logging): keep a content-block system prompt in logged messages (#51699)
* fix(logging): keep a content-block system prompt in logged messages A /v1/messages request whose top-level system prompt is a list of content blocks never reached the logged messages, because only a plain-text system prompt was prepended. Accept a non-empty list too, so every logger that reads messages sees it. Claude on Vertex also normalizes a plain-text system prompt into blocks, so it was dropped there as well. Fixes #36402 * fix(logging): type the logged system message * fix(logging): type the system message without a mutable annotation, close the test client
This commit is contained in:
parent
d11917e89c
commit
0f97850c50
2 changed files with 91 additions and 20 deletions
|
|
@ -201,6 +201,7 @@ from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
|||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
Batch,
|
||||
ChatCompletionSystemMessage,
|
||||
FineTuningJob,
|
||||
HttpxBinaryResponseContent,
|
||||
OpenAIFileObject,
|
||||
|
|
@ -7404,27 +7405,21 @@ class StandardLoggingPayloadSetup:
|
|||
@staticmethod
|
||||
def append_system_prompt_messages(kwargs: dict | None = None, messages: Any | None = None):
|
||||
"""
|
||||
Append system prompt messages to the messages
|
||||
Prepend the request's top-level `system` prompt, text or content blocks, to the logged messages
|
||||
"""
|
||||
if kwargs is not None:
|
||||
if kwargs.get("system") is not None and isinstance(kwargs.get("system"), str):
|
||||
if messages is None:
|
||||
return [{"role": "system", "content": kwargs.get("system")}]
|
||||
elif isinstance(messages, list):
|
||||
if len(messages) == 0:
|
||||
return [{"role": "system", "content": kwargs.get("system")}]
|
||||
# check for duplicates
|
||||
if messages[0].get("role") == "system" and messages[0].get("content") == kwargs.get("system"):
|
||||
return messages
|
||||
messages = [{"role": "system", "content": kwargs.get("system")}] + messages
|
||||
elif isinstance(messages, str):
|
||||
messages = [
|
||||
{"role": "system", "content": kwargs.get("system")},
|
||||
{"role": "user", "content": messages},
|
||||
]
|
||||
return messages
|
||||
|
||||
return messages
|
||||
system: Final = kwargs.get("system") if kwargs is not None else None
|
||||
if not isinstance(system, str) and not (isinstance(system, list) and system):
|
||||
return messages
|
||||
system_message: Final = ChatCompletionSystemMessage(role="system", content=system)
|
||||
if messages is None:
|
||||
return [system_message]
|
||||
if isinstance(messages, str):
|
||||
return [system_message, {"role": "user", "content": messages}]
|
||||
if not isinstance(messages, list):
|
||||
return messages
|
||||
if messages and messages[0].get("role") == "system" and messages[0].get("content") == system:
|
||||
return messages
|
||||
return [system_message, *messages]
|
||||
|
||||
@staticmethod
|
||||
def merge_litellm_metadata(litellm_params: Mapping[str, object]) -> dict[str, object]:
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ from litellm.cost_calculator import ocr_batch_cost
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_logger_v2 import Abort, CallContext, CustomLoggerV2, Field, Hook, Patch, on
|
||||
from litellm.integrations.custom_logger_v2.host import V2Outcome
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.custom_llm import CustomLLM, CustomLLMError
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
|
||||
|
|
@ -4817,6 +4818,81 @@ def test_append_system_prompt_messages():
|
|||
assert result == messages
|
||||
|
||||
|
||||
_SYSTEM_BLOCKS: Final = ({"type": "text", "text": "You are a code reviewer.", "cache_control": {"type": "ephemeral"}},)
|
||||
_USER_TURN: Final = MappingProxyType({"role": "user", "content": "Say hi"})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"system, messages",
|
||||
[
|
||||
([], [dict(_USER_TURN)]),
|
||||
(list(_SYSTEM_BLOCKS), [{"role": "system", "content": list(_SYSTEM_BLOCKS)}, dict(_USER_TURN)]),
|
||||
],
|
||||
ids=["empty_block_list", "blocks_already_first"],
|
||||
)
|
||||
def test_logged_messages_gain_no_extra_system_entry(
|
||||
system: list[dict[str, object]], messages: list[dict[str, object]]
|
||||
) -> None:
|
||||
result: Final = StandardLoggingPayloadSetup.append_system_prompt_messages(
|
||||
kwargs={"system": system}, messages=messages
|
||||
)
|
||||
assert result == messages
|
||||
|
||||
|
||||
class _LoggedMessagesRecorder(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.messages: object = None
|
||||
self.logged: asyncio.Event = asyncio.Event()
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, StandardLoggingPayload], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self.messages = kwargs["standard_logging_object"]["messages"]
|
||||
self.logged.set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"system",
|
||||
[
|
||||
"You are a code reviewer.",
|
||||
[{"type": "text", "text": "You are a code reviewer.", "cache_control": {"type": "ephemeral"}}],
|
||||
],
|
||||
ids=["text", "content_blocks"],
|
||||
)
|
||||
async def test_anthropic_messages_system_prompt_reaches_logged_messages(system: str | list[dict[str, object]]) -> None:
|
||||
recorder: Final = _LoggedMessagesRecorder()
|
||||
user_turn: Final = {"role": "user", "content": "Say hi"}
|
||||
reply: Final = {
|
||||
"id": "msg_system_prompt_logging",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": [{"type": "text", "text": "hi"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 5, "output_tokens": 1},
|
||||
}
|
||||
upstream: Final = AsyncHTTPHandler(
|
||||
transport=httpx.MockTransport(lambda request: httpx.Response(200, json=reply, request=request))
|
||||
)
|
||||
try:
|
||||
await litellm.anthropic.messages.acreate(
|
||||
model="anthropic/claude-sonnet-5-5",
|
||||
max_tokens=16,
|
||||
system=system,
|
||||
messages=[user_turn],
|
||||
api_key="sk-test",
|
||||
client=upstream,
|
||||
callbacks=[recorder],
|
||||
)
|
||||
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
|
||||
finally:
|
||||
await upstream.close()
|
||||
assert recorder.messages == [{"role": "system", "content": system}, user_turn]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_success_handler_sets_standard_logging_object_for_pass_through_endpoints():
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue