mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-01 02:02:20 +00:00
fix(compact_20260112): use beta-header constant; require type discriminator; skip sync bridge when idle
- bedrock: replace hardcoded "compact-2026-01-12" beta string with ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value in both Converse (_filter_context_management_for_bedrock_converse) and Invoke (anthropic_claude3) compact-edit handlers. - types: mark the "type" discriminator as Required[...] on the new CompactionBlock and UsageIteration TypedDicts so the discriminator is not silently optional under total=False. - adapters/handler: short-circuit the sync /v1/messages adapter path before spawning the run_async_function worker-thread event loop when the request has no context_management spec and no client-sent compaction block in the message history. Test plan: - uv run pytest tests/test_litellm/llms/anthropic/experimental_pass_through/ tests/test_litellm/llms/bedrock/test_converse_context_management.py -q (370 + 10 = 380 passed) - uv run pytest tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_transformation.py tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py -k compact (3 passed)
This commit is contained in:
parent
2cc4ef2a97
commit
bdddff0c36
4 changed files with 44 additions and 17 deletions
|
|
@ -40,6 +40,18 @@ if TYPE_CHECKING:
|
|||
ANTHROPIC_ONLY_REQUEST_KEYS: frozenset[str] = frozenset({"output_config"})
|
||||
|
||||
|
||||
def _messages_have_compaction_block(messages: List[Dict]) -> bool:
|
||||
"""Return True when any message carries a ``compaction`` content block."""
|
||||
for msg in messages:
|
||||
content = msg.get("content")
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "compaction":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def _prepare_context_managed_request(
|
||||
*,
|
||||
model: str,
|
||||
|
|
@ -677,17 +689,26 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
# unaffected because it ``await``s within the original event loop.
|
||||
litellm_router = kwargs.pop("litellm_router", None)
|
||||
|
||||
polyfill_result = run_async_function(
|
||||
_prepare_context_managed_request,
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
system=system,
|
||||
context_management_spec=context_management,
|
||||
metadata=metadata,
|
||||
drop_params=drop_params,
|
||||
llm_router=litellm_router,
|
||||
)
|
||||
# Skip the async bridge entirely when there is nothing for either the
|
||||
# polyfill or the client-history slice-only fallback to do. The vast
|
||||
# majority of sync ``litellm.messages.create()`` requests carry no
|
||||
# ``context_management`` spec and no client-sent ``compaction`` block,
|
||||
# and bridging through a worker-thread event loop just to discover
|
||||
# there is no work is pure overhead.
|
||||
if context_management is None and not _messages_have_compaction_block(messages):
|
||||
polyfill_result: Optional[PolyfillResult] = None
|
||||
else:
|
||||
polyfill_result = run_async_function(
|
||||
_prepare_context_managed_request,
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
system=system,
|
||||
context_management_spec=context_management,
|
||||
metadata=metadata,
|
||||
drop_params=drop_params,
|
||||
llm_router=litellm_router,
|
||||
)
|
||||
|
||||
effective_messages = (
|
||||
polyfill_result.messages if polyfill_result is not None else messages
|
||||
|
|
|
|||
|
|
@ -1477,6 +1477,8 @@ class AmazonConverseConfig(BaseConfig):
|
|||
anthropic_beta_list: list,
|
||||
) -> None:
|
||||
"""Keep only compact_20260112 edits for Bedrock; add beta header or drop field."""
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
||||
cm = additional_request_params.get("context_management")
|
||||
if not isinstance(cm, dict):
|
||||
additional_request_params.pop("context_management", None)
|
||||
|
|
@ -1492,8 +1494,9 @@ class AmazonConverseConfig(BaseConfig):
|
|||
if isinstance(e, dict) and e.get("type") == "compact_20260112"
|
||||
]
|
||||
if compact_edits:
|
||||
if "compact-2026-01-12" not in anthropic_beta_list:
|
||||
anthropic_beta_list.append("compact-2026-01-12")
|
||||
compact_beta = ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value
|
||||
if compact_beta not in anthropic_beta_list:
|
||||
anthropic_beta_list.append(compact_beta)
|
||||
additional_request_params["context_management"] = {
|
||||
**cm,
|
||||
"edits": compact_edits,
|
||||
|
|
|
|||
|
|
@ -38,7 +38,10 @@ from litellm.llms.bedrock.common_utils import (
|
|||
normalize_tool_input_schema_types_for_bedrock_invoke,
|
||||
remove_custom_field_from_tools,
|
||||
)
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
|
||||
from litellm.types.llms.anthropic import (
|
||||
ANTHROPIC_BETA_HEADER_VALUES,
|
||||
ANTHROPIC_TOOL_SEARCH_BETA_HEADER,
|
||||
)
|
||||
from litellm.types.llms.bedrock import BedrockInvokeAnthropicMessagesRequest
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
|
@ -442,7 +445,7 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
if isinstance(e, dict) and e.get("type") == "compact_20260112"
|
||||
]
|
||||
if compact_edits:
|
||||
beta_set.add("compact-2026-01-12")
|
||||
beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
|
||||
anthropic_messages_request["context_management"] = {
|
||||
**cm,
|
||||
"edits": compact_edits,
|
||||
|
|
|
|||
|
|
@ -537,14 +537,14 @@ class ContextManagementResponse(TypedDict, total=False):
|
|||
class CompactionBlock(TypedDict, total=False):
|
||||
"""Synthesized ``compaction`` content block (compact_20260112)."""
|
||||
|
||||
type: Literal["compaction"]
|
||||
type: Required[Literal["compaction"]]
|
||||
content: Optional[str]
|
||||
|
||||
|
||||
class UsageIteration(TypedDict, total=False):
|
||||
"""One sampling iteration's token usage (compact_20260112)."""
|
||||
|
||||
type: Literal["compaction", "message"]
|
||||
type: Required[Literal["compaction", "message"]]
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue