scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

Merge a6082c93dc into 0c98afa780

Browse source
This commit is contained in:
elyar1124 2026-09-13 02:04:47 +08:00 • committed by GitHub
commit 4ff764fc2b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 129 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
litellm/llms/anthropic/chat/handler.py
View file

@ -48,7 +48,6 @@ from litellm.types.responses.main import (
)
from litellm.types.utils import (
Delta,
GenericStreamingChunk,
LlmProviders,
ModelResponse,
ModelResponseStream,
@ -1261,14 +1260,7 @@ class ModelResponseIterator:
return result
# If None, continue loop to get more chunks for accumulation
else:
return GenericStreamingChunk(
text="",
is_finished=False,
finish_reason="",
usage=None,
index=0,
tool_use=None,
)
continue
except StopIteration:
raise StopIteration
except ValueError as e:
@ -1307,14 +1299,7 @@ class ModelResponseIterator:
return result
# If None, continue loop to get more chunks for accumulation
else:
return GenericStreamingChunk(
text="",
is_finished=False,
finish_reason="",
usage=None,
index=0,
tool_use=None,
)
continue
except StopAsyncIteration:
raise StopAsyncIteration
except ValueError as e:

127
tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py
View file

@ -2704,3 +2704,130 @@ class TestRustChatCompletionsHook:
"model": "m",
"messages": [],
}
def test_anthropic_streaming_thinking_text_tool_use_sync():
sse_lines = [
b'event: message_start\n',
b'data: {"type":"message_start","message":{"id":"msg_test_123","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}\n',
b'\n',
b'event: content_block_start\n',
b'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}\n',
b'\n',
b'event: content_block_delta\n',
b'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Let me call submit_result"}}\n',
b'\n',
b'event: content_block_stop\n',
b'data: {"type":"content_block_stop","index":0}\n',
b'\n',
b'event: content_block_start\n',
b'data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}\n',
b'\n',
b'event: content_block_delta\n',
b'data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"I will now submit the result."}}\n',
b'\n',
b'event: content_block_stop\n',
b'data: {"type":"content_block_stop","index":1}\n',
b'\n',
b'event: content_block_start\n',
b'data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_01","name":"submit_result","input":{}}}\n',
b'\n',
b'event: content_block_delta\n',
b'data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\\"value\\":"}}\n',
b'\n',
b'event: content_block_delta\n',
b'data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"1}"}}\n',
b'\n',
b'event: content_block_stop\n',
b'data: {"type":"content_block_stop","index":2}\n',
b'\n',
b'event: message_delta\n',
b'data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":25}}\n',
b'\n',
b'event: message_stop\n',
b'data: {"type":"message_stop"}\n',
]
iterator = ModelResponseIterator(
streaming_response=iter(sse_lines),
sync_stream=True,
json_mode=False,
)
chunks = list(iterator)
assert all(isinstance(c, litellm.ModelResponseStream) for c in chunks)
tool_call_chunks = [
c for c in chunks
if getattr(c.choices[0].delta, "tool_calls", None) is not None
]
assert len(tool_call_chunks) == 3
assert tool_call_chunks[0].choices[0].delta.tool_calls[0].function.name == "submit_result"
assert tool_call_chunks[0].choices[0].delta.tool_calls[0].id == "toolu_01"
built = litellm.stream_chunk_builder(chunks=chunks)
assert built.choices[0].message.content == "I will now submit the result."
assert built.choices[0].message.tool_calls is not None
assert len(built.choices[0].message.tool_calls) == 1
assert built.choices[0].message.tool_calls[0].function.name == "submit_result"
assert built.choices[0].message.tool_calls[0].function.arguments == '{"value":1}'
assert built.choices[0].finish_reason == "tool_calls"
@pytest.mark.asyncio
async def test_anthropic_streaming_thinking_text_tool_use_async():
sse_lines = [
'event: message_start\n',
'data: {"type":"message_start","message":{"id":"msg_async_123","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}\n',
'event: content_block_start\n',
'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}\n',
'event: content_block_delta\n',
'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Async thinking..."}}\n',
'event: content_block_stop\n',
'data: {"type":"content_block_stop","index":0}\n',
'event: content_block_start\n',
'data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_async_01","name":"do_action","input":{}}}\n',
'event: content_block_delta\n',
'data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\\"action\\": \\"run\\"}"}}\n',
'event: content_block_stop\n',
'data: {"type":"content_block_stop","index":1}\n',
'event: message_delta\n',
'data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":15}}\n',
'event: message_stop\n',
'data: {"type":"message_stop"}\n',
]
class AsyncLineIterator:
def __init__(self, lines):
self._iter = iter(lines)
def __aiter__(self):
return self
async def __anext__(self):
try:
return next(self._iter)
except StopIteration:
raise StopAsyncIteration
iterator = ModelResponseIterator(
streaming_response=AsyncLineIterator(sse_lines),
sync_stream=False,
json_mode=False,
)
chunks = []
async for chunk in iterator:
chunks.append(chunk)
assert all(isinstance(c, litellm.ModelResponseStream) for c in chunks)
tool_call_chunks = [
c for c in chunks
if getattr(c.choices[0].delta, "tool_calls", None) is not None
]
assert len(tool_call_chunks) == 2
assert tool_call_chunks[0].choices[0].delta.tool_calls[0].function.name == "do_action"
built = litellm.stream_chunk_builder(chunks=chunks)
assert built.choices[0].message.tool_calls[0].function.name == "do_action"
assert json.loads(built.choices[0].message.tool_calls[0].function.arguments) == {"action": "run"}