diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index 0e4fa8c8eeb..5a6f2507dda 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -10,13 +10,14 @@ from itertools import groupby from typing import Any, Final, TypeAlias from openai.lib import _parsing, _pydantic -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import TypeIs # noqa: TID251 # TypeIs lands in typing only on 3.13 from litellm._logging import verbose_logger from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.types.llms.openai import ( AllMessageValues, + ChatCompletionCachedContent, ChatCompletionSystemMessage, ChatCompletionTextObject, ChatCompletionToolCallChunk, @@ -215,11 +216,27 @@ def type_to_response_format_param( SystemMessageContent: TypeAlias = str | list[object] -_system_content_adapter: Final = TypeAdapter[SystemMessageContent](SystemMessageContent) +_system_content_adapter: Final = TypeAdapter[SystemMessageContent | None](SystemMessageContent | None) +_content_block_adapter: Final = TypeAdapter[dict[str, object]](dict[str, object]) def _system_content(message: ChatCompletionSystemMessage) -> SystemMessageContent: - return _system_content_adapter.validate_python(message["content"]) + content: Final = _system_content_adapter.validate_python(message.get("content")) + return "" if content is None else content + + +def _with_message_cache_control( + blocks: list[object], cache_control: ChatCompletionCachedContent | None +) -> list[object]: # mutable-ok: block lists are the wire format + if not blocks or cache_control is None: + return blocks + try: + last: Final = _content_block_adapter.validate_python(blocks[-1]) + except ValidationError: + return blocks + if "cache_control" in last: + return blocks + return [*blocks[:-1], {**last, "cache_control": cache_control}] # mutable-ok: block lists are the wire format def _as_system_message(message: AllMessageValues) -> AllMessageValues: @@ -241,11 +258,11 @@ def map_developer_role_to_system_role( def _text_blocks(message: ChatCompletionSystemMessage) -> list[object]: # mutable-ok: block lists are the wire format content: Final = _system_content(message) + cache_control: Final = message.get("cache_control") if not isinstance(content, str): - return content + return _with_message_cache_control(content, cache_control) if not content: return [] # mutable-ok: block lists are the wire format - cache_control: Final = message.get("cache_control") if cache_control: cached_block: Final[ChatCompletionTextObject] = { "type": "text", diff --git a/litellm/llms/litellm_proxy/chat/transformation.py b/litellm/llms/litellm_proxy/chat/transformation.py index 26dfb5221ca..05856656c1e 100644 --- a/litellm/llms/litellm_proxy/chat/transformation.py +++ b/litellm/llms/litellm_proxy/chat/transformation.py @@ -2,8 +2,7 @@ Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions` """ -from collections.abc import Sequence -from typing import TYPE_CHECKING, Final +from typing import Final from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS from litellm.secret_managers.main import get_secret_bool, get_secret_str @@ -11,24 +10,8 @@ from litellm.types.router import LiteLLM_Params from ...openai.chat.gpt_transformation import OpenAIGPTConfig -if TYPE_CHECKING: - from litellm.types.llms.openai import AllMessageValues - class LiteLLMProxyChatConfig(OpenAIGPTConfig): - def translate_developer_role_to_system_role( - self, - messages: Sequence["AllMessageValues"], - *, - custom_llm_provider: str | None, - api_base: str | None, - ) -> Sequence["AllMessageValues"]: - """ - The downstream LiteLLM proxy translates developer messages for whichever - backend it routes to, so they pass through untouched. - """ - return messages - def get_supported_openai_params(self, model: str) -> list: params_list: Final = super().get_supported_openai_params(model) params_list.extend(OPENAI_CHAT_COMPLETION_PARAMS) diff --git a/tests/test_litellm/llms/base_llm/test_base_utils.py b/tests/test_litellm/llms/base_llm/test_base_utils.py index d9a24f0b244..7e9effb14e1 100644 --- a/tests/test_litellm/llms/base_llm/test_base_utils.py +++ b/tests/test_litellm/llms/base_llm/test_base_utils.py @@ -358,6 +358,61 @@ class TestHoistDeveloperMessagesIntoLeadingSystemMessage: {"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]}, ] + def test_system_message_without_content_folds_into_the_run_instead_of_raising(self): + messages = [ + {"role": "system", "content": "Rules"}, + {"role": "system"}, + {"role": "system", "content": None}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Rules"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_message_level_cache_control_of_a_block_content_member_lands_on_its_last_block(self): + messages = [ + { + "role": "system", + "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}], + "cache_control": {"type": "ephemeral"}, + }, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Rule 1"}, + {"type": "text", "text": "Rule 2", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "Answer with exactly one word."}, + ], + }, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_message_level_cache_control_does_not_override_a_block_that_already_carries_one(self): + messages = [ + { + "role": "system", + "content": [{"type": "text", "text": "Rule 1", "cache_control": {"type": "ephemeral", "ttl": "1h"}}], + "cache_control": {"type": "ephemeral"}, + }, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Rule 1", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + {"type": "text", "text": "Answer with exactly one word."}, + ], + }, + {"role": "user", "content": "What is the capital of France?"}, + ] + def test_non_consecutive_native_system_messages_stay_where_the_client_put_them(self): messages = [ {"role": "system", "content": "System turn 1"}, diff --git a/tests/test_litellm/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py b/tests/test_litellm/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py index 3b89f03e6b1..6864db65960 100644 --- a/tests/test_litellm/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py +++ b/tests/test_litellm/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py @@ -42,7 +42,7 @@ def test_litellm_gateway_from_sdk_with_user_param(): assert "user" in supported_params -def test_translate_developer_role_passes_developer_messages_through_to_the_downstream_proxy(): +def test_translate_developer_role_hoists_a_later_developer_message_before_the_downstream_proxy_sees_it(): messages = [ {"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi there"}, @@ -56,4 +56,8 @@ def test_translate_developer_role_passes_developer_messages_through_to_the_downs api_base="http://inner-proxy:4000", ) - assert list(translated) == messages + assert list(translated) == [ + {"role": "system", "content": "You are terse.\n\nAnswer with exactly one word."}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ]