This commit is contained in:
Alon Sokolovski 2026-10-27 02:24:24 +00:00 • committed by GitHub
commit 2b22bce043
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 365 additions and 9 deletions

View file

@ -10,10 +10,14 @@ from typing import Final, TypeAlias
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
supports_openai_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.asyncify import (
run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
from litellm.responses.utils import auto_reasoning_summary
@ -24,6 +28,7 @@ from litellm.types.llms.anthropic import (
AnthropicMessagesRequest,
AnthropicOutputConfig,
AnthropicOutputSchema,
ContextManagementResponse,
)
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
@ -31,7 +36,10 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.utils import ProviderConfigManager
from ..adapters.handler import request_safeguards_evaluator
from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator
from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management
from ..context_management.constants import COMPACT_EDIT_TYPE
from ..context_management.dispatcher import normalize_spec
from ..convertible_blocks import reject_malformed_tool_result_text
from ..safeguards import with_safeguard_results
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
@ -41,10 +49,73 @@ from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]]
BlockRewrite: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]]
_BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
# (native spec for the Responses API, edits for the gateway polyfill)
_SplitEdits: TypeAlias = tuple[dict[str, object] | None, Sequence[Mapping[str, object]]]
_ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
def _split_context_management(
spec: dict[str, object] | None, # mutable-ok: the request's spec, handed to normalize_spec
kwargs: Mapping[str, object],
) -> _SplitEdits:
"""Return (the edits the Responses API runs natively, the edits the gateway polyfill runs).
The Responses API compacts natively (``compact_20260112`` becomes its ``compaction``); any other
edit, such as ``clear_tool_uses_20250919``, has no Responses API form and runs in the polyfill, as
it does on the chat completions path.
"""
if not spec or context_management_explicitly_dropped(kwargs.get("additional_drop_params")):
return spec, ()
edits: Final = normalize_spec(spec) or []
native: Final = [edit for edit in edits if edit.get("type") == COMPACT_EDIT_TYPE]
return ({"edits": native} if native else None), tuple(
edit for edit in edits if edit.get("type") != COMPACT_EDIT_TYPE
)
async def _polyfilled(
*,
model: str,
messages: AnthropicRequestMessages,
tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None,
system: str | None,
edits: Sequence[Mapping[str, object]],
kwargs: Mapping[str, object],
) -> PolyfillResult | None:
"""Run the edits the Responses API can't run natively; raise like the chat completions path does."""
if not edits:
return None
drop_invalid_edits: Final = drops_unsupported_params(kwargs.get("drop_params"))
try:
return await apply_context_management(
model=model,
messages=_BLOCK_LIST.validate_python(messages),
tools=_BLOCK_LIST.validate_python(tools) if tools else None,
system=system,
context_management_spec={"edits": [dict(edit) for edit in edits]},
drop_invalid_edits=drop_invalid_edits,
)
except AnthropicContextManagementError:
if drop_invalid_edits:
return None
raise
except Exception as e:
verbose_logger.exception("context_management polyfill failed on the Responses API path: %s", e)
raise AnthropicContextManagementError(
status_code=500, message=f"context_management polyfill failed: {e}"
) from e
def _with_applied_edits(
response: AnthropicMessagesResponse, polyfill_result: PolyfillResult | None
) -> AnthropicMessagesResponse:
applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
if not applied_edits:
return response
return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))}
def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
if logging_obj is not None:
@ -278,11 +349,16 @@ class LiteLLMMessagesToResponsesAPIHandler:
output_format: AnthropicOutputSchema | None = None,
**kwargs: object,
) -> AnthropicMessagesResponse | AsyncIterator[bytes]:
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
polyfill_result: Final = await _polyfilled(
model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs
)
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=messages,
# clear_tool_uses edits only rewrite messages; system stays as sent
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
context_management=context_management,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
@ -307,13 +383,16 @@ class LiteLLMMessagesToResponsesAPIHandler:
model=local_model_name(model, kwargs.get("custom_llm_provider")),
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
safeguards_evaluator=safeguards_evaluator,
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
served: Final = await with_safeguard_results(
_with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator
)
_record_bridged_response(responses_kwargs, result)
return served
@ -364,11 +443,28 @@ class LiteLLMMessagesToResponsesAPIHandler:
)
# Sync path
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
ran: Final[object] = ( # pyright: ignore[reportUnknownVariableType] # run_async_function is untyped
run_async_function(
_polyfilled,
model=model,
messages=messages,
tools=tools,
system=system,
edits=polyfilled_edits,
kwargs=kwargs,
)
if polyfilled_edits
else None
)
# run_async_function is untyped: check what it returned rather than assume it
polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=messages,
# clear_tool_uses edits only rewrite messages; system stays as sent
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
context_management=context_management,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
@ -391,10 +487,11 @@ class LiteLLMMessagesToResponsesAPIHandler:
responses_stream=result,
model=local_model_name(model, kwargs.get("custom_llm_provider")),
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
return _ADAPTER.translate_response(result)
return _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result)

View file

@ -2,7 +2,7 @@
## Translates OpenAI call to Anthropic `/v1/messages` format
import asyncio
from collections import deque
from collections.abc import AsyncGenerator, Iterator, Mapping
from collections.abc import AsyncGenerator, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
from pydantic import ConfigDict, field_validator
@ -25,6 +25,7 @@ from litellm.responses.streaming_iterator import (
partial_responses_incomplete_event_on_disconnect,
stream_error_status_and_message,
)
from litellm.types.llms.anthropic import AppliedEdit, ContextManagementResponse
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from litellm.types.llms.base import LiteLLMBaseModel
@ -127,8 +128,11 @@ class AnthropicResponsesStreamWrapper:
model: str,
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
applied_edits: Sequence[AppliedEdit] | None = None,
) -> None:
self.responses_stream = responses_stream
# The gateway polyfill's edits, reported on the final message_delta as on the chat completions path
self._applied_edits: Final = tuple(applied_edits or ())
self.model = model
self.safeguards_evaluator: Final = safeguards_evaluator
self._message_id: str = f"msg_{uuid.uuid4()}"
@ -433,6 +437,11 @@ class AnthropicResponsesStreamWrapper:
"type": "message_delta",
"delta": message_delta_payload,
"usage": dict(anthropic_usage),
**(
{"context_management": ContextManagementResponse(applied_edits=list(self._applied_edits))}
if self._applied_edits
else {}
),
}
)
self._chunk_queue.append({"type": "message_stop"})

View file

@ -2,7 +2,7 @@ import datetime
import json
import os
import sys
from collections.abc import Mapping
from collections.abc import AsyncIterator, Mapping
from typing import Final, cast
from unittest.mock import AsyncMock, patch
@ -739,3 +739,253 @@ async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_
assert logging_obj.cost_breakdown is not None
assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output)
assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning)
TEXT_RESPONSE: Final = {
"id": "resp_cleared",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "gpt-5.6-luna",
"output": [
{
"type": "message",
"id": "msg_1",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "done", "annotations": []}],
}
],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
"usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
}
CLEAR_EDIT: Final = {
"type": "clear_tool_uses_20250919",
"trigger": {"type": "tool_uses", "value": 1},
"keep": {"type": "tool_uses", "value": 1},
}
COMPACT_EDIT: Final = {"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}
def _three_reads() -> list[dict[str, object]]:
messages: Final[list[dict[str, object]]] = [{"role": "user", "content": "Read the logs."}]
for i in range(3):
messages.append(
{
"role": "assistant",
"content": [{"type": "tool_use", "id": f"toolu_{i}", "name": "Bash", "input": {"command": f"cat {i}"}}],
}
)
messages.append(
{
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": f"toolu_{i}", "content": f"log {i} " * 40}],
}
)
return messages
async def _cleared_bridge(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, *, stream: bool, edits: list[dict[str, object]]
) -> tuple[object, Mapping[str, object]]:
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses")
if stream:
upstream.respond(status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY)
else:
upstream.respond(status_code=200, json=TEXT_RESPONSE)
response: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=1024,
messages=_three_reads(),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
stream=stream,
custom_llm_provider="openai",
context_management={"edits": edits},
)
body: Final = json.loads(upstream.calls.last.request.content)
if not stream:
return response, body
return [json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in response], body
@pytest.mark.asyncio
async def test_clear_tool_uses_clears_old_results_before_the_responses_api_call(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.llms.anthropic.pass_through.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER
served, body = await _cleared_bridge(respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT)])
outputs: Final = {
item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output"
}
assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [
True,
True,
False,
]
assert "context_management" not in body
assert isinstance(served, dict)
assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
@pytest.mark.asyncio
async def test_streamed_clear_tool_uses_reports_its_edits_on_the_final_message_delta(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
events, _ = await _cleared_bridge(respx_mock, monkeypatch, stream=True, edits=[dict(CLEAR_EDIT)])
assert isinstance(events, list)
message_delta: Final = next(event for event in events if event["type"] == "message_delta")
assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
@pytest.mark.asyncio
async def test_compact_stays_native_while_clear_tool_uses_runs_in_the_gateway(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
served, body = await _cleared_bridge(
respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT), dict(COMPACT_EDIT)]
)
assert body["context_management"] == [{"type": "compaction", "compact_threshold": 150000}]
assert isinstance(served, dict)
assert [edit["type"] for edit in served["context_management"]["applied_edits"]] == ["clear_tool_uses_20250919"]
@pytest.mark.asyncio
async def test_a_malformed_clear_tool_uses_edit_is_a_400_before_the_responses_api_call(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.llms.anthropic.pass_through.context_management import AnthropicContextManagementError
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
status_code=200, json=TEXT_RESPONSE
)
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
with pytest.raises(AnthropicContextManagementError) as raised:
await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=1024,
messages=_three_reads(),
model="openai/gpt-5.6-luna",
custom_llm_provider="openai",
context_management={"edits": [{**CLEAR_EDIT, "exclude_tools": "Bash"}]},
)
assert raised.value.status_code == 400
assert not upstream.called
def test_the_sync_path_clears_old_results_too(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.llms.anthropic.pass_through.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
status_code=200, json=TEXT_RESPONSE
)
served: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=1024,
messages=_three_reads(),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
custom_llm_provider="openai",
context_management={"edits": [dict(CLEAR_EDIT)]},
)
body: Final = json.loads(upstream.calls.last.request.content)
outputs: Final = {
item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output"
}
assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [
True,
True,
False,
]
assert isinstance(served, dict)
assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
@pytest.mark.asyncio
async def test_a_polyfill_crash_is_a_500_before_the_responses_api_call(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.llms.anthropic.pass_through.context_management import AnthropicContextManagementError
from litellm.llms.anthropic.pass_through.responses_adapters import handler
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
status_code=200, json=TEXT_RESPONSE
)
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
monkeypatch.setattr(handler, "apply_context_management", AsyncMock(side_effect=RuntimeError("editor bug")))
with pytest.raises(AnthropicContextManagementError) as raised:
await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=1024,
messages=_three_reads(),
model="openai/gpt-5.6-luna",
custom_llm_provider="openai",
context_management={"edits": [dict(CLEAR_EDIT)]},
)
assert raised.value.status_code == 500
assert not upstream.called
@pytest.mark.asyncio
async def test_under_drop_params_a_rejected_edit_is_skipped_and_the_request_goes_out(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.llms.anthropic.pass_through.context_management import AnthropicContextManagementError
from litellm.llms.anthropic.pass_through.responses_adapters import handler
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
status_code=200, json=TEXT_RESPONSE
)
rejected: Final = AnthropicContextManagementError(status_code=400, message="bad edit")
monkeypatch.setattr(handler, "apply_context_management", AsyncMock(side_effect=rejected))
served: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=1024,
messages=_three_reads(),
model="openai/gpt-5.6-luna",
custom_llm_provider="openai",
context_management={"edits": [dict(CLEAR_EDIT)]},
drop_params=True,
)
assert upstream.called
assert isinstance(served, dict) and "context_management" not in served
@pytest.mark.asyncio
async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
respx_mock.post("https://api.openai.com/v1/responses").respond(
status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY
)
sse: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=1024,
messages=_three_reads(),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
stream=True,
custom_llm_provider="openai",
context_management={"edits": [dict(CLEAR_EDIT)]},
)
events: Final = [
json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in cast(AsyncIterator[bytes], sse)
]
message_delta: Final = next(event for event in events if event["type"] == "message_delta")
assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2