mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
fix(anthropic/messages): apply context_management on sync path; clear held stop_reason chunk in async iterator
- Sync `anthropic_messages_handler` was silently dropping the `context_management` kwarg via `ANTHROPIC_ONLY_REQUEST_KEYS` after the polyfill was moved into the async handler. Bridge to the async dispatcher with `run_async_function` so `litellm.messages.create()` callers keep working (regressed e.g. `clear_tool_uses_20250919`). - In the streaming iterator's `__anext__` `StopIteration` handler, clear `self.holding_stop_reason_chunk` after capturing it (matches `__next__`) so a subsequent call doesn't re-emit the same chunk. Co-authored-by: Yassin Kortam <yassin@berri.ai>
This commit is contained in:
parent
6fdf82414d
commit
15ea941fbe
37 changed files with 46 additions and 4 deletions
|
|
@ -13,6 +13,7 @@ from typing import (
|
||||||
|
|
||||||
import litellm
|
import litellm
|
||||||
from litellm._logging import verbose_logger
|
from litellm._logging import verbose_logger
|
||||||
|
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||||
from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import (
|
from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import (
|
||||||
AnthropicAdapter,
|
AnthropicAdapter,
|
||||||
)
|
)
|
||||||
|
|
@ -477,17 +478,52 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
||||||
**kwargs,
|
**kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Run the context_management polyfill on the sync path too so that
|
||||||
|
# ``litellm.messages.create()`` callers don't silently lose edits like
|
||||||
|
# ``clear_tool_uses_20250919``. The dispatcher is async (so the
|
||||||
|
# ``compact_20260112`` editor can ``await`` the summarization model);
|
||||||
|
# bridge to it via ``run_async_function``.
|
||||||
|
context_management = kwargs.pop("context_management", None)
|
||||||
|
drop_params: Optional[bool] = kwargs.get("drop_params", None)
|
||||||
|
litellm_router = kwargs.pop("litellm_router", None)
|
||||||
|
if litellm_router is None:
|
||||||
|
try:
|
||||||
|
from litellm.proxy.proxy_server import llm_router as _proxy_router
|
||||||
|
|
||||||
|
litellm_router = _proxy_router
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
polyfill_result = run_async_function(
|
||||||
|
_run_polyfill_if_enabled,
|
||||||
|
model=model,
|
||||||
|
messages=messages,
|
||||||
|
tools=tools,
|
||||||
|
system=system,
|
||||||
|
context_management_spec=context_management,
|
||||||
|
metadata=metadata,
|
||||||
|
drop_params=drop_params,
|
||||||
|
llm_router=litellm_router,
|
||||||
|
)
|
||||||
|
|
||||||
|
effective_messages = (
|
||||||
|
polyfill_result.messages if polyfill_result is not None else messages
|
||||||
|
)
|
||||||
|
effective_system = (
|
||||||
|
polyfill_result.system if polyfill_result is not None else system
|
||||||
|
)
|
||||||
|
|
||||||
(
|
(
|
||||||
completion_kwargs,
|
completion_kwargs,
|
||||||
tool_name_mapping,
|
tool_name_mapping,
|
||||||
) = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs(
|
) = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs(
|
||||||
max_tokens=max_tokens,
|
max_tokens=max_tokens,
|
||||||
messages=messages,
|
messages=effective_messages,
|
||||||
model=model,
|
model=model,
|
||||||
metadata=metadata,
|
metadata=metadata,
|
||||||
stop_sequences=stop_sequences,
|
stop_sequences=stop_sequences,
|
||||||
stream=stream,
|
stream=stream,
|
||||||
system=system,
|
system=effective_system,
|
||||||
temperature=temperature,
|
temperature=temperature,
|
||||||
thinking=thinking,
|
thinking=thinking,
|
||||||
tool_choice=tool_choice,
|
tool_choice=tool_choice,
|
||||||
|
|
@ -506,6 +542,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
||||||
completion_response,
|
completion_response,
|
||||||
model=model,
|
model=model,
|
||||||
tool_name_mapping=tool_name_mapping,
|
tool_name_mapping=tool_name_mapping,
|
||||||
|
polyfill_result=polyfill_result,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if transformed_stream is not None:
|
if transformed_stream is not None:
|
||||||
|
|
@ -515,6 +552,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
||||||
anthropic_response = ANTHROPIC_ADAPTER.translate_completion_output_params(
|
anthropic_response = ANTHROPIC_ADAPTER.translate_completion_output_params(
|
||||||
cast(ModelResponse, completion_response),
|
cast(ModelResponse, completion_response),
|
||||||
tool_name_mapping=tool_name_mapping,
|
tool_name_mapping=tool_name_mapping,
|
||||||
|
polyfill_result=polyfill_result,
|
||||||
)
|
)
|
||||||
if anthropic_response is not None:
|
if anthropic_response is not None:
|
||||||
return anthropic_response
|
return anthropic_response
|
||||||
|
|
|
||||||
|
|
@ -521,9 +521,13 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
||||||
# Handle any remaining queued chunks before stopping
|
# Handle any remaining queued chunks before stopping
|
||||||
if self.chunk_queue:
|
if self.chunk_queue:
|
||||||
return self.chunk_queue.popleft()
|
return self.chunk_queue.popleft()
|
||||||
# Handle any held stop_reason chunk
|
# Handle any held stop_reason chunk — clear after capturing so a
|
||||||
|
# subsequent ``__anext__`` call doesn't re-emit the same chunk
|
||||||
|
# (matches the sync ``__next__`` path).
|
||||||
if self.holding_stop_reason_chunk is not None:
|
if self.holding_stop_reason_chunk is not None:
|
||||||
return self.holding_stop_reason_chunk
|
held = self.holding_stop_reason_chunk
|
||||||
|
self.holding_stop_reason_chunk = None
|
||||||
|
return held
|
||||||
if not self.sent_last_message:
|
if not self.sent_last_message:
|
||||||
self.sent_last_message = True
|
self.sent_last_message = True
|
||||||
return {"type": "message_stop"}
|
return {"type": "message_stop"}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue