scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-12 08:42:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 207

fix(vertex_ai): log why context caching is skipped and note the developer-role boundary

Browse source
This commit is contained in:
Mrun25 2026-10-17 12:07:04 +05:30
parent b58633fbd1
commit da7f912d04
1 changed files with 13 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
litellm/llms/vertex_ai/context_caching/transformation.py
View file

@ -10,6 +10,7 @@ from datetime import datetime, timezone
from types import MappingProxyType
from typing import Final, Literal
from litellm._logging import verbose_logger
from litellm.types.llms.openai import AllMessageValues
from litellm.types.llms.vertex_ai import CachedContentRequestBody
from litellm.utils import is_cached_message
@ -147,9 +148,20 @@ def separate_cached_messages(
prefix_messages: Final = messages[:first_cached_idx]
suffix_messages: Final = messages[last_cached_idx + 1 :]
if any(m["role"] != "system" for m in prefix_messages) or any(m["role"] == "system" for m in suffix_messages):
# Only "system" counts as foldable: the Vertex/Gemini mapping does not
# fold "developer" messages into system_instruction, so a developer
# message around the block intentionally disables caching.
non_system_prefix: Final = any(m["role"] != "system" for m in prefix_messages)
trailing_system: Final = any(m["role"] == "system" for m in suffix_messages)
if non_system_prefix or trailing_system:
# Not a true prefix: send the conversation uncached so nothing is
# reordered and no system message is silently dropped.
verbose_logger.debug(
"Vertex context caching skipped: cached block is not a true prefix (%s)",
"non-system message before the cached block"
if non_system_prefix
else "system message after the cached block",
)
return [], list(messages)
cached_messages = list(prefix_messages) + messages[first_cached_idx : last_cached_idx + 1]