mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-01 02:02:20 +00:00
fix: address bug detection findings in token counter and streaming iterator
- token_counter: guard against non-dict 'function' field in tool dicts and skip tools missing a name to avoid emitting 'type None = ...' which would produce inaccurate token counts. - streaming_iterator: change sync __next__ generic-error path to raise StopIteration (was StopAsyncIteration), so sync iteration cleanly stops. - streaming_iterator: centralize context_management attachment so the held-stop_reason direct-flush path defensively re-attaches applied_edits to match the merge path's guarantee. Co-authored-by: Yassin Kortam <yassin@berri.ai>
This commit is contained in:
parent
d50ea4325a
commit
aad5df50a0
2 changed files with 35 additions and 4 deletions
|
|
@ -767,7 +767,7 @@ def _format_function_definitions(tools):
|
|||
if not isinstance(tool, dict):
|
||||
continue
|
||||
function = tool.get("function")
|
||||
if function is None:
|
||||
if not isinstance(function, dict):
|
||||
# Anthropic tool shape → OpenAI function dict for token counting.
|
||||
params = tool.get("input_schema") or tool.get("parameters") or {}
|
||||
if not isinstance(params, dict):
|
||||
|
|
@ -777,9 +777,13 @@ def _format_function_definitions(tools):
|
|||
"description": tool.get("description"),
|
||||
"parameters": params,
|
||||
}
|
||||
function_name = function.get("name")
|
||||
if not function_name:
|
||||
# Skip malformed tools missing a name to avoid emitting
|
||||
# ``type None = ...`` which would produce inaccurate token counts.
|
||||
continue
|
||||
if function_description := function.get("description"):
|
||||
lines.append(f"// {function_description}")
|
||||
function_name = function.get("name")
|
||||
parameters = function.get("parameters") or {}
|
||||
if not isinstance(parameters, dict):
|
||||
parameters = {}
|
||||
|
|
|
|||
|
|
@ -130,10 +130,37 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
)
|
||||
return self._augment_message_delta_usage(merged_chunk)
|
||||
|
||||
def _ensure_context_management_attached(
|
||||
self, message_delta_chunk: Dict[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
"""Attach ``context_management`` to a ``message_delta`` chunk if
|
||||
``self.applied_edits`` is non-empty and the chunk does not already
|
||||
carry it. Returns the (possibly new) chunk dict.
|
||||
|
||||
Centralizing this guard ensures every ``message_delta`` emission
|
||||
path (merge-with-usage and direct-flush-of-held) consistently
|
||||
surfaces ``applied_edits`` to the client.
|
||||
"""
|
||||
if not self.applied_edits or "context_management" in message_delta_chunk:
|
||||
return message_delta_chunk
|
||||
augmented = message_delta_chunk.copy()
|
||||
augmented["context_management"] = ContextManagementResponse(
|
||||
applied_edits=list(self.applied_edits)
|
||||
)
|
||||
return augmented
|
||||
|
||||
def _augment_message_delta_usage(
|
||||
self, message_delta_chunk: Dict[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
"""Attach polyfill compaction iteration usage to the final message_delta."""
|
||||
"""Attach polyfill compaction iteration usage to the final message_delta.
|
||||
|
||||
Also defensively re-attaches ``context_management`` so the direct
|
||||
held-chunk flush path stays in sync with the merge path's guarantee
|
||||
when ``self.applied_edits`` is non-empty.
|
||||
"""
|
||||
message_delta_chunk = self._ensure_context_management_attached(
|
||||
message_delta_chunk
|
||||
)
|
||||
if self.iterations_usage is None:
|
||||
return message_delta_chunk
|
||||
usage = message_delta_chunk.get("usage")
|
||||
|
|
@ -433,7 +460,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
verbose_logger.error(
|
||||
"Anthropic Adapter - {}\n{}".format(e, traceback.format_exc())
|
||||
)
|
||||
raise StopAsyncIteration
|
||||
raise StopIteration
|
||||
|
||||
async def __anext__(self): # noqa: PLR0915
|
||||
from .transformation import LiteLLMAnthropicMessagesAdapter
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue