scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 129

fix(openai): hoist developer messages for litellm_proxy and harden the system fold

Browse source 
- drop the LiteLLMProxyChatConfig passthrough so litellm_proxy/ hoists later developer messages like every other OpenAI-compatible endpoint, whichever backend or LiteLLM version sits downstream
- a system message with null or missing content inside a folded run is dropped instead of raising KeyError('content')
- a message-level cache_control on a list-content member lands on its last block so the breakpoint survives the fold
This commit is contained in:
mateo-berri 2026-09-05 03:40:47 -07:00
parent 69fd884493
commit ea840eb6dc
4 changed files with 84 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
litellm/llms/base_llm/base_utils.py
View file

@ -10,13 +10,14 @@ from itertools import groupby
from typing import Any, Final, TypeAlias
from openai.lib import _parsing, _pydantic
from pydantic import BaseModel, TypeAdapter
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import TypeIs # noqa: TID251 # TypeIs lands in typing only on 3.13
from litellm._logging import verbose_logger
from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionCachedContent,
ChatCompletionSystemMessage,
ChatCompletionTextObject,
ChatCompletionToolCallChunk,
@ -215,11 +216,27 @@ def type_to_response_format_param(
SystemMessageContent: TypeAlias = str | list[object]
_system_content_adapter: Final = TypeAdapter[SystemMessageContent](SystemMessageContent)
_system_content_adapter: Final = TypeAdapter[SystemMessageContent | None](SystemMessageContent | None)
_content_block_adapter: Final = TypeAdapter[dict[str, object]](dict[str, object])
def _system_content(message: ChatCompletionSystemMessage) -> SystemMessageContent:
return _system_content_adapter.validate_python(message["content"])
content: Final = _system_content_adapter.validate_python(message.get("content"))
return "" if content is None else content
def _with_message_cache_control(
blocks: list[object], cache_control: ChatCompletionCachedContent | None
) -> list[object]: # mutable-ok: block lists are the wire format
if not blocks or cache_control is None:
return blocks
try:
last: Final = _content_block_adapter.validate_python(blocks[-1])
except ValidationError:
return blocks
if "cache_control" in last:
return blocks
return [*blocks[:-1], {**last, "cache_control": cache_control}] # mutable-ok: block lists are the wire format
def _as_system_message(message: AllMessageValues) -> AllMessageValues:
@ -241,11 +258,11 @@ def map_developer_role_to_system_role(
def _text_blocks(message: ChatCompletionSystemMessage) -> list[object]: # mutable-ok: block lists are the wire format
content: Final = _system_content(message)
cache_control: Final = message.get("cache_control")
if not isinstance(content, str):
return content
return _with_message_cache_control(content, cache_control)
if not content:
return [] # mutable-ok: block lists are the wire format
cache_control: Final = message.get("cache_control")
if cache_control:
cached_block: Final[ChatCompletionTextObject] = {
"type": "text",

19
litellm/llms/litellm_proxy/chat/transformation.py
View file

@ -2,8 +2,7 @@
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
"""
from collections.abc import Sequence
from typing import TYPE_CHECKING, Final
from typing import Final
from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
from litellm.secret_managers.main import get_secret_bool, get_secret_str
@ -11,24 +10,8 @@ from litellm.types.router import LiteLLM_Params
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
if TYPE_CHECKING:
from litellm.types.llms.openai import AllMessageValues
class LiteLLMProxyChatConfig(OpenAIGPTConfig):
def translate_developer_role_to_system_role(
self,
messages: Sequence["AllMessageValues"],
*,
custom_llm_provider: str | None,
api_base: str | None,
) -> Sequence["AllMessageValues"]:
"""
The downstream LiteLLM proxy translates developer messages for whichever
backend it routes to, so they pass through untouched.
"""
return messages
def get_supported_openai_params(self, model: str) -> list:
params_list: Final = super().get_supported_openai_params(model)
params_list.extend(OPENAI_CHAT_COMPLETION_PARAMS)

55
tests/test_litellm/llms/base_llm/test_base_utils.py
View file

@ -358,6 +358,61 @@ class TestHoistDeveloperMessagesIntoLeadingSystemMessage:
{"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]},
]
def test_system_message_without_content_folds_into_the_run_instead_of_raising(self):
messages = [
{"role": "system", "content": "Rules"},
{"role": "system"},
{"role": "system", "content": None},
{"role": "user", "content": "What is the capital of France?"},
]
assert list(hoist_developer_messages_into_leading_system_message(messages)) == [
{"role": "system", "content": "Rules"},
{"role": "user", "content": "What is the capital of France?"},
]
def test_message_level_cache_control_of_a_block_content_member_lands_on_its_last_block(self):
messages = [
{
"role": "system",
"content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}],
"cache_control": {"type": "ephemeral"},
},
{"role": "developer", "content": "Answer with exactly one word."},
{"role": "user", "content": "What is the capital of France?"},
]
assert list(hoist_developer_messages_into_leading_system_message(messages)) == [
{
"role": "system",
"content": [
{"type": "text", "text": "Rule 1"},
{"type": "text", "text": "Rule 2", "cache_control": {"type": "ephemeral"}},
{"type": "text", "text": "Answer with exactly one word."},
],
},
{"role": "user", "content": "What is the capital of France?"},
]
def test_message_level_cache_control_does_not_override_a_block_that_already_carries_one(self):
messages = [
{
"role": "system",
"content": [{"type": "text", "text": "Rule 1", "cache_control": {"type": "ephemeral", "ttl": "1h"}}],
"cache_control": {"type": "ephemeral"},
},
{"role": "developer", "content": "Answer with exactly one word."},
{"role": "user", "content": "What is the capital of France?"},
]
assert list(hoist_developer_messages_into_leading_system_message(messages)) == [
{
"role": "system",
"content": [
{"type": "text", "text": "Rule 1", "cache_control": {"type": "ephemeral", "ttl": "1h"}},
{"type": "text", "text": "Answer with exactly one word."},
],
},
{"role": "user", "content": "What is the capital of France?"},
]
def test_non_consecutive_native_system_messages_stay_where_the_client_put_them(self):
messages = [
{"role": "system", "content": "System turn 1"},

8
tests/test_litellm/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py
View file

@ -42,7 +42,7 @@ def test_litellm_gateway_from_sdk_with_user_param():
assert "user" in supported_params
def test_translate_developer_role_passes_developer_messages_through_to_the_downstream_proxy():
def test_translate_developer_role_hoists_a_later_developer_message_before_the_downstream_proxy_sees_it():
messages = [
{"role": "system", "content": "You are terse."},
{"role": "user", "content": "Hi there"},
@ -56,4 +56,8 @@ def test_translate_developer_role_passes_developer_messages_through_to_the_downs
api_base="http://inner-proxy:4000",
)
assert list(translated) == messages
assert list(translated) == [
{"role": "system", "content": "You are terse.\n\nAnswer with exactly one word."},
{"role": "user", "content": "Hi there"},
{"role": "user", "content": "What is the capital of France?"},
]