fix(logging): keep a content-block system prompt in logged messages (#51699)

* fix(logging): keep a content-block system prompt in logged messages

A /v1/messages request whose top-level system prompt is a list of content
blocks never reached the logged messages, because only a plain-text system
prompt was prepended. Accept a non-empty list too, so every logger that reads
messages sees it. Claude on Vertex also normalizes a plain-text system prompt
into blocks, so it was dropped there as well.

Fixes #36402

* fix(logging): type the logged system message

* fix(logging): type the system message without a mutable annotation, close the test client
This commit is contained in:
ryan-crabbe-berri 2026-11-13 16:23:29 -08:00 • committed by GitHub
parent d11917e89c
commit 0f97850c50
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 91 additions and 20 deletions

View file

@ -201,6 +201,7 @@ from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
from litellm.types.llms.openai import (
AllMessageValues,
Batch,
ChatCompletionSystemMessage,
FineTuningJob,
HttpxBinaryResponseContent,
OpenAIFileObject,
@ -7404,27 +7405,21 @@ class StandardLoggingPayloadSetup:
@staticmethod
def append_system_prompt_messages(kwargs: dict | None = None, messages: Any | None = None):
"""
Append system prompt messages to the messages
Prepend the request's top-level `system` prompt, text or content blocks, to the logged messages
"""
if kwargs is not None:
if kwargs.get("system") is not None and isinstance(kwargs.get("system"), str):
if messages is None:
return [{"role": "system", "content": kwargs.get("system")}]
elif isinstance(messages, list):
if len(messages) == 0:
return [{"role": "system", "content": kwargs.get("system")}]
# check for duplicates
if messages[0].get("role") == "system" and messages[0].get("content") == kwargs.get("system"):
return messages
messages = [{"role": "system", "content": kwargs.get("system")}] + messages
elif isinstance(messages, str):
messages = [
{"role": "system", "content": kwargs.get("system")},
{"role": "user", "content": messages},
]
return messages
return messages
system: Final = kwargs.get("system") if kwargs is not None else None
if not isinstance(system, str) and not (isinstance(system, list) and system):
return messages
system_message: Final = ChatCompletionSystemMessage(role="system", content=system)
if messages is None:
return [system_message]
if isinstance(messages, str):
return [system_message, {"role": "user", "content": messages}]
if not isinstance(messages, list):
return messages
if messages and messages[0].get("role") == "system" and messages[0].get("content") == system:
return messages
return [system_message, *messages]
@staticmethod
def merge_litellm_metadata(litellm_params: Mapping[str, object]) -> dict[str, object]:

View file

@ -42,6 +42,7 @@ from litellm.cost_calculator import ocr_batch_cost
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_logger_v2 import Abort, CallContext, CustomLoggerV2, Field, Hook, Patch, on
from litellm.integrations.custom_logger_v2.host import V2Outcome
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.custom_llm import CustomLLM, CustomLLMError
from litellm.litellm_core_utils import litellm_logging
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
@ -4817,6 +4818,81 @@ def test_append_system_prompt_messages():
assert result == messages
_SYSTEM_BLOCKS: Final = ({"type": "text", "text": "You are a code reviewer.", "cache_control": {"type": "ephemeral"}},)
_USER_TURN: Final = MappingProxyType({"role": "user", "content": "Say hi"})
@pytest.mark.parametrize(
"system, messages",
[
([], [dict(_USER_TURN)]),
(list(_SYSTEM_BLOCKS), [{"role": "system", "content": list(_SYSTEM_BLOCKS)}, dict(_USER_TURN)]),
],
ids=["empty_block_list", "blocks_already_first"],
)
def test_logged_messages_gain_no_extra_system_entry(
system: list[dict[str, object]], messages: list[dict[str, object]]
) -> None:
result: Final = StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs={"system": system}, messages=messages
)
assert result == messages
class _LoggedMessagesRecorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.messages: object = None
self.logged: asyncio.Event = asyncio.Event()
async def async_log_success_event(
self, kwargs: Mapping[str, StandardLoggingPayload], response_obj: object, start_time: object, end_time: object
) -> None:
self.messages = kwargs["standard_logging_object"]["messages"]
self.logged.set()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"system",
[
"You are a code reviewer.",
[{"type": "text", "text": "You are a code reviewer.", "cache_control": {"type": "ephemeral"}}],
],
ids=["text", "content_blocks"],
)
async def test_anthropic_messages_system_prompt_reaches_logged_messages(system: str | list[dict[str, object]]) -> None:
recorder: Final = _LoggedMessagesRecorder()
user_turn: Final = {"role": "user", "content": "Say hi"}
reply: Final = {
"id": "msg_system_prompt_logging",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 5, "output_tokens": 1},
}
upstream: Final = AsyncHTTPHandler(
transport=httpx.MockTransport(lambda request: httpx.Response(200, json=reply, request=request))
)
try:
await litellm.anthropic.messages.acreate(
model="anthropic/claude-sonnet-5-5",
max_tokens=16,
system=system,
messages=[user_turn],
api_key="sk-test",
client=upstream,
callbacks=[recorder],
)
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
finally:
await upstream.close()
assert recorder.messages == [{"role": "system", "content": system}, user_turn]
@pytest.mark.asyncio
async def test_async_success_handler_sets_standard_logging_object_for_pass_through_endpoints():
"""