mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 898e45585e into e16d79d635
This commit is contained in:
commit
2b22bce043
3 changed files with 365 additions and 9 deletions
|
|
@ -10,10 +10,14 @@ from typing import Final, TypeAlias
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
supports_openai_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import (
|
||||
run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
|
||||
from litellm.responses.utils import auto_reasoning_summary
|
||||
|
|
@ -24,6 +28,7 @@ from litellm.types.llms.anthropic import (
|
|||
AnthropicMessagesRequest,
|
||||
AnthropicOutputConfig,
|
||||
AnthropicOutputSchema,
|
||||
ContextManagementResponse,
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
|
|
@ -31,7 +36,10 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
|
|||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
from ..adapters.handler import request_safeguards_evaluator
|
||||
from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator
|
||||
from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management
|
||||
from ..context_management.constants import COMPACT_EDIT_TYPE
|
||||
from ..context_management.dispatcher import normalize_spec
|
||||
from ..convertible_blocks import reject_malformed_tool_result_text
|
||||
from ..safeguards import with_safeguard_results
|
||||
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
|
||||
|
|
@ -41,10 +49,73 @@ from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
|
|||
AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]]
|
||||
BlockRewrite: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]]
|
||||
_BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
|
||||
# (native spec for the Responses API, edits for the gateway polyfill)
|
||||
_SplitEdits: TypeAlias = tuple[dict[str, object] | None, Sequence[Mapping[str, object]]]
|
||||
|
||||
_ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
|
||||
|
||||
|
||||
def _split_context_management(
|
||||
spec: dict[str, object] | None, # mutable-ok: the request's spec, handed to normalize_spec
|
||||
kwargs: Mapping[str, object],
|
||||
) -> _SplitEdits:
|
||||
"""Return (the edits the Responses API runs natively, the edits the gateway polyfill runs).
|
||||
|
||||
The Responses API compacts natively (``compact_20260112`` becomes its ``compaction``); any other
|
||||
edit, such as ``clear_tool_uses_20250919``, has no Responses API form and runs in the polyfill, as
|
||||
it does on the chat completions path.
|
||||
"""
|
||||
if not spec or context_management_explicitly_dropped(kwargs.get("additional_drop_params")):
|
||||
return spec, ()
|
||||
edits: Final = normalize_spec(spec) or []
|
||||
native: Final = [edit for edit in edits if edit.get("type") == COMPACT_EDIT_TYPE]
|
||||
return ({"edits": native} if native else None), tuple(
|
||||
edit for edit in edits if edit.get("type") != COMPACT_EDIT_TYPE
|
||||
)
|
||||
|
||||
|
||||
async def _polyfilled(
|
||||
*,
|
||||
model: str,
|
||||
messages: AnthropicRequestMessages,
|
||||
tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None,
|
||||
system: str | None,
|
||||
edits: Sequence[Mapping[str, object]],
|
||||
kwargs: Mapping[str, object],
|
||||
) -> PolyfillResult | None:
|
||||
"""Run the edits the Responses API can't run natively; raise like the chat completions path does."""
|
||||
if not edits:
|
||||
return None
|
||||
drop_invalid_edits: Final = drops_unsupported_params(kwargs.get("drop_params"))
|
||||
try:
|
||||
return await apply_context_management(
|
||||
model=model,
|
||||
messages=_BLOCK_LIST.validate_python(messages),
|
||||
tools=_BLOCK_LIST.validate_python(tools) if tools else None,
|
||||
system=system,
|
||||
context_management_spec={"edits": [dict(edit) for edit in edits]},
|
||||
drop_invalid_edits=drop_invalid_edits,
|
||||
)
|
||||
except AnthropicContextManagementError:
|
||||
if drop_invalid_edits:
|
||||
return None
|
||||
raise
|
||||
except Exception as e:
|
||||
verbose_logger.exception("context_management polyfill failed on the Responses API path: %s", e)
|
||||
raise AnthropicContextManagementError(
|
||||
status_code=500, message=f"context_management polyfill failed: {e}"
|
||||
) from e
|
||||
|
||||
|
||||
def _with_applied_edits(
|
||||
response: AnthropicMessagesResponse, polyfill_result: PolyfillResult | None
|
||||
) -> AnthropicMessagesResponse:
|
||||
applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
|
||||
if not applied_edits:
|
||||
return response
|
||||
return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))}
|
||||
|
||||
|
||||
def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
|
||||
logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
|
||||
if logging_obj is not None:
|
||||
|
|
@ -278,11 +349,16 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
output_format: AnthropicOutputSchema | None = None,
|
||||
**kwargs: object,
|
||||
) -> AnthropicMessagesResponse | AsyncIterator[bytes]:
|
||||
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
|
||||
polyfill_result: Final = await _polyfilled(
|
||||
model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs
|
||||
)
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
# clear_tool_uses edits only rewrite messages; system stays as sent
|
||||
messages=polyfill_result.messages if polyfill_result is not None else messages,
|
||||
model=model,
|
||||
context_management=context_management,
|
||||
context_management=native_context_management,
|
||||
metadata=metadata,
|
||||
output_config=output_config,
|
||||
stop_sequences=stop_sequences,
|
||||
|
|
@ -307,13 +383,16 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
model=local_model_name(model, kwargs.get("custom_llm_provider")),
|
||||
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
|
||||
safeguards_evaluator=safeguards_evaluator,
|
||||
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
|
||||
)
|
||||
return wrapper.async_anthropic_sse_wrapper()
|
||||
|
||||
if not isinstance(result, ResponsesAPIResponse):
|
||||
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
|
||||
|
||||
served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
|
||||
served: Final = await with_safeguard_results(
|
||||
_with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator
|
||||
)
|
||||
_record_bridged_response(responses_kwargs, result)
|
||||
return served
|
||||
|
||||
|
|
@ -364,11 +443,28 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
)
|
||||
|
||||
# Sync path
|
||||
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
|
||||
ran: Final[object] = ( # pyright: ignore[reportUnknownVariableType] # run_async_function is untyped
|
||||
run_async_function(
|
||||
_polyfilled,
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
system=system,
|
||||
edits=polyfilled_edits,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
if polyfilled_edits
|
||||
else None
|
||||
)
|
||||
# run_async_function is untyped: check what it returned rather than assume it
|
||||
polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
# clear_tool_uses edits only rewrite messages; system stays as sent
|
||||
messages=polyfill_result.messages if polyfill_result is not None else messages,
|
||||
model=model,
|
||||
context_management=context_management,
|
||||
context_management=native_context_management,
|
||||
metadata=metadata,
|
||||
output_config=output_config,
|
||||
stop_sequences=stop_sequences,
|
||||
|
|
@ -391,10 +487,11 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
responses_stream=result,
|
||||
model=local_model_name(model, kwargs.get("custom_llm_provider")),
|
||||
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
|
||||
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
|
||||
)
|
||||
return wrapper.async_anthropic_sse_wrapper()
|
||||
|
||||
if not isinstance(result, ResponsesAPIResponse):
|
||||
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
|
||||
|
||||
return _ADAPTER.translate_response(result)
|
||||
return _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
## Translates OpenAI call to Anthropic `/v1/messages` format
|
||||
import asyncio
|
||||
from collections import deque
|
||||
from collections.abc import AsyncGenerator, Iterator, Mapping
|
||||
from collections.abc import AsyncGenerator, Iterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from pydantic import ConfigDict, field_validator
|
||||
|
|
@ -25,6 +25,7 @@ from litellm.responses.streaming_iterator import (
|
|||
partial_responses_incomplete_event_on_disconnect,
|
||||
stream_error_status_and_message,
|
||||
)
|
||||
from litellm.types.llms.anthropic import AppliedEdit, ContextManagementResponse
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
|
|
@ -127,8 +128,11 @@ class AnthropicResponsesStreamWrapper:
|
|||
model: str,
|
||||
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
|
||||
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
|
||||
applied_edits: Sequence[AppliedEdit] | None = None,
|
||||
) -> None:
|
||||
self.responses_stream = responses_stream
|
||||
# The gateway polyfill's edits, reported on the final message_delta as on the chat completions path
|
||||
self._applied_edits: Final = tuple(applied_edits or ())
|
||||
self.model = model
|
||||
self.safeguards_evaluator: Final = safeguards_evaluator
|
||||
self._message_id: str = f"msg_{uuid.uuid4()}"
|
||||
|
|
@ -433,6 +437,11 @@ class AnthropicResponsesStreamWrapper:
|
|||
"type": "message_delta",
|
||||
"delta": message_delta_payload,
|
||||
"usage": dict(anthropic_usage),
|
||||
**(
|
||||
{"context_management": ContextManagementResponse(applied_edits=list(self._applied_edits))}
|
||||
if self._applied_edits
|
||||
else {}
|
||||
),
|
||||
}
|
||||
)
|
||||
self._chunk_queue.append({"type": "message_stop"})
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import datetime
|
|||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from typing import Final, cast
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
|
|
@ -739,3 +739,253 @@ async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_
|
|||
assert logging_obj.cost_breakdown is not None
|
||||
assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output)
|
||||
assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning)
|
||||
|
||||
|
||||
TEXT_RESPONSE: Final = {
|
||||
"id": "resp_cleared",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "gpt-5.6-luna",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "done", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": True,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
|
||||
}
|
||||
CLEAR_EDIT: Final = {
|
||||
"type": "clear_tool_uses_20250919",
|
||||
"trigger": {"type": "tool_uses", "value": 1},
|
||||
"keep": {"type": "tool_uses", "value": 1},
|
||||
}
|
||||
COMPACT_EDIT: Final = {"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}
|
||||
|
||||
|
||||
def _three_reads() -> list[dict[str, object]]:
|
||||
messages: Final[list[dict[str, object]]] = [{"role": "user", "content": "Read the logs."}]
|
||||
for i in range(3):
|
||||
messages.append(
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{"type": "tool_use", "id": f"toolu_{i}", "name": "Bash", "input": {"command": f"cat {i}"}}],
|
||||
}
|
||||
)
|
||||
messages.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "tool_result", "tool_use_id": f"toolu_{i}", "content": f"log {i} " * 40}],
|
||||
}
|
||||
)
|
||||
return messages
|
||||
|
||||
|
||||
async def _cleared_bridge(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, *, stream: bool, edits: list[dict[str, object]]
|
||||
) -> tuple[object, Mapping[str, object]]:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses")
|
||||
if stream:
|
||||
upstream.respond(status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY)
|
||||
else:
|
||||
upstream.respond(status_code=200, json=TEXT_RESPONSE)
|
||||
response: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
|
||||
max_tokens=1024,
|
||||
messages=_three_reads(),
|
||||
model="openai/gpt-5.6-luna",
|
||||
tools=[dict(BASH_TOOL)],
|
||||
stream=stream,
|
||||
custom_llm_provider="openai",
|
||||
context_management={"edits": edits},
|
||||
)
|
||||
body: Final = json.loads(upstream.calls.last.request.content)
|
||||
if not stream:
|
||||
return response, body
|
||||
return [json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in response], body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_tool_uses_clears_old_results_before_the_responses_api_call(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.pass_through.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER
|
||||
|
||||
served, body = await _cleared_bridge(respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT)])
|
||||
|
||||
outputs: Final = {
|
||||
item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output"
|
||||
}
|
||||
assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
]
|
||||
assert "context_management" not in body
|
||||
assert isinstance(served, dict)
|
||||
assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_clear_tool_uses_reports_its_edits_on_the_final_message_delta(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
events, _ = await _cleared_bridge(respx_mock, monkeypatch, stream=True, edits=[dict(CLEAR_EDIT)])
|
||||
|
||||
assert isinstance(events, list)
|
||||
message_delta: Final = next(event for event in events if event["type"] == "message_delta")
|
||||
assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_stays_native_while_clear_tool_uses_runs_in_the_gateway(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
served, body = await _cleared_bridge(
|
||||
respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT), dict(COMPACT_EDIT)]
|
||||
)
|
||||
|
||||
assert body["context_management"] == [{"type": "compaction", "compact_threshold": 150000}]
|
||||
assert isinstance(served, dict)
|
||||
assert [edit["type"] for edit in served["context_management"]["applied_edits"]] == ["clear_tool_uses_20250919"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_malformed_clear_tool_uses_edit_is_a_400_before_the_responses_api_call(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.pass_through.context_management import AnthropicContextManagementError
|
||||
|
||||
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
|
||||
status_code=200, json=TEXT_RESPONSE
|
||||
)
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
|
||||
with pytest.raises(AnthropicContextManagementError) as raised:
|
||||
await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
|
||||
max_tokens=1024,
|
||||
messages=_three_reads(),
|
||||
model="openai/gpt-5.6-luna",
|
||||
custom_llm_provider="openai",
|
||||
context_management={"edits": [{**CLEAR_EDIT, "exclude_tools": "Bash"}]},
|
||||
)
|
||||
assert raised.value.status_code == 400
|
||||
assert not upstream.called
|
||||
|
||||
|
||||
def test_the_sync_path_clears_old_results_too(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.llms.anthropic.pass_through.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER
|
||||
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
|
||||
status_code=200, json=TEXT_RESPONSE
|
||||
)
|
||||
|
||||
served: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
|
||||
max_tokens=1024,
|
||||
messages=_three_reads(),
|
||||
model="openai/gpt-5.6-luna",
|
||||
tools=[dict(BASH_TOOL)],
|
||||
custom_llm_provider="openai",
|
||||
context_management={"edits": [dict(CLEAR_EDIT)]},
|
||||
)
|
||||
|
||||
body: Final = json.loads(upstream.calls.last.request.content)
|
||||
outputs: Final = {
|
||||
item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output"
|
||||
}
|
||||
assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
]
|
||||
assert isinstance(served, dict)
|
||||
assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_polyfill_crash_is_a_500_before_the_responses_api_call(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.pass_through.context_management import AnthropicContextManagementError
|
||||
from litellm.llms.anthropic.pass_through.responses_adapters import handler
|
||||
|
||||
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
|
||||
status_code=200, json=TEXT_RESPONSE
|
||||
)
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
|
||||
monkeypatch.setattr(handler, "apply_context_management", AsyncMock(side_effect=RuntimeError("editor bug")))
|
||||
with pytest.raises(AnthropicContextManagementError) as raised:
|
||||
await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
|
||||
max_tokens=1024,
|
||||
messages=_three_reads(),
|
||||
model="openai/gpt-5.6-luna",
|
||||
custom_llm_provider="openai",
|
||||
context_management={"edits": [dict(CLEAR_EDIT)]},
|
||||
)
|
||||
assert raised.value.status_code == 500
|
||||
assert not upstream.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_under_drop_params_a_rejected_edit_is_skipped_and_the_request_goes_out(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.pass_through.context_management import AnthropicContextManagementError
|
||||
from litellm.llms.anthropic.pass_through.responses_adapters import handler
|
||||
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond(
|
||||
status_code=200, json=TEXT_RESPONSE
|
||||
)
|
||||
rejected: Final = AnthropicContextManagementError(status_code=400, message="bad edit")
|
||||
monkeypatch.setattr(handler, "apply_context_management", AsyncMock(side_effect=rejected))
|
||||
|
||||
served: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
|
||||
max_tokens=1024,
|
||||
messages=_three_reads(),
|
||||
model="openai/gpt-5.6-luna",
|
||||
custom_llm_provider="openai",
|
||||
context_management={"edits": [dict(CLEAR_EDIT)]},
|
||||
drop_params=True,
|
||||
)
|
||||
assert upstream.called
|
||||
assert isinstance(served, dict) and "context_management" not in served
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test")
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
respx_mock.post("https://api.openai.com/v1/responses").respond(
|
||||
status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY
|
||||
)
|
||||
|
||||
sse: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
|
||||
max_tokens=1024,
|
||||
messages=_three_reads(),
|
||||
model="openai/gpt-5.6-luna",
|
||||
tools=[dict(BASH_TOOL)],
|
||||
stream=True,
|
||||
custom_llm_provider="openai",
|
||||
context_management={"edits": [dict(CLEAR_EDIT)]},
|
||||
)
|
||||
events: Final = [
|
||||
json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in cast(AsyncIterator[bytes], sse)
|
||||
]
|
||||
message_delta: Final = next(event for event in events if event["type"] == "message_delta")
|
||||
assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue