From efaca405743045f9911e537e2bed7de26942d837 Mon Sep 17 00:00:00 2001 From: elyar1124 Date: Tue, 1 Sep 2026 16:12:11 +0800 Subject: [PATCH 1/4] fix(anthropic): skip non-data SSE lines in stream iterator to preserve streamed tool calls --- litellm/llms/anthropic/chat/handler.py | 19 +-- .../chat/test_anthropic_chat_handler.py | 141 ++++++++++++++++++ 2 files changed, 143 insertions(+), 17 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index c82be07a5c5..bf59833ebe1 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -47,7 +47,6 @@ from litellm.types.responses.main import ( ) from litellm.types.utils import ( Delta, - GenericStreamingChunk, LlmProviders, ModelResponse, ModelResponseStream, @@ -1255,14 +1254,7 @@ class ModelResponseIterator: return result # If None, continue loop to get more chunks for accumulation else: - return GenericStreamingChunk( - text="", - is_finished=False, - finish_reason="", - usage=None, - index=0, - tool_use=None, - ) + continue except StopIteration: raise StopIteration except ValueError as e: @@ -1301,14 +1293,7 @@ class ModelResponseIterator: return result # If None, continue loop to get more chunks for accumulation else: - return GenericStreamingChunk( - text="", - is_finished=False, - finish_reason="", - usage=None, - index=0, - tool_use=None, - ) + continue except StopAsyncIteration: raise StopAsyncIteration except ValueError as e: diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index bd750a47f63..0ac0d2573ab 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -2568,3 +2568,144 @@ class TestRustChatCompletionsHook: "model": "m", "messages": [], } + + +def test_anthropic_streaming_thinking_text_tool_use_sync(): + """ + Regression test for Issue #36262: + Ensure Anthropic SSE stream containing thinking -> text -> tool_use emits only + ModelResponseStream instances (no dummy GenericStreamingChunk dicts), + preserves delta.tool_calls, and successfully builds via stream_chunk_builder. + """ + sse_lines = [ + b'event: message_start\n', + b'data: {"type":"message_start","message":{"id":"msg_test_123","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}\n', + b'\n', + b'event: content_block_start\n', + b'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}\n', + b'\n', + b'event: content_block_delta\n', + b'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Let me call submit_result"}}\n', + b'\n', + b'event: content_block_stop\n', + b'data: {"type":"content_block_stop","index":0}\n', + b'\n', + b'event: content_block_start\n', + b'data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}\n', + b'\n', + b'event: content_block_delta\n', + b'data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"I will now submit the result."}}\n', + b'\n', + b'event: content_block_stop\n', + b'data: {"type":"content_block_stop","index":1}\n', + b'\n', + b'event: content_block_start\n', + b'data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_01","name":"submit_result","input":{}}}\n', + b'\n', + b'event: content_block_delta\n', + b'data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\\"value\\":"}}\n', + b'\n', + b'event: content_block_delta\n', + b'data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"1}"}}\n', + b'\n', + b'event: content_block_stop\n', + b'data: {"type":"content_block_stop","index":2}\n', + b'\n', + b'event: message_delta\n', + b'data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":25}}\n', + b'\n', + b'event: message_stop\n', + b'data: {"type":"message_stop"}\n', + ] + + iterator = ModelResponseIterator( + streaming_response=iter(sse_lines), + sync_stream=True, + json_mode=False, + ) + chunks = list(iterator) + + # All emitted chunks must be ModelResponseStream instances + assert all(isinstance(c, litellm.ModelResponseStream) for c in chunks) + + # Tool call chunks must be present + tool_call_chunks = [ + c for c in chunks + if getattr(c.choices[0].delta, "tool_calls", None) is not None + ] + assert len(tool_call_chunks) == 3 + assert tool_call_chunks[0].choices[0].delta.tool_calls[0].function.name == "submit_result" + assert tool_call_chunks[0].choices[0].delta.tool_calls[0].id == "toolu_01" + + # stream_chunk_builder must reconstruct the message with tool calls + built = litellm.stream_chunk_builder(chunks=chunks) + assert built.choices[0].message.content == "I will now submit the result." + assert built.choices[0].message.tool_calls is not None + assert len(built.choices[0].message.tool_calls) == 1 + assert built.choices[0].message.tool_calls[0].function.name == "submit_result" + assert built.choices[0].message.tool_calls[0].function.arguments == '{"value":1}' + assert built.choices[0].finish_reason == "tool_calls" + + +@pytest.mark.asyncio +async def test_anthropic_streaming_thinking_text_tool_use_async(): + """ + Regression test for Issue #36262 in async streaming (__anext__): + Ensure async iteration correctly yields all ModelResponseStream chunks and preserves tool_calls. + """ + sse_lines = [ + 'event: message_start\n', + 'data: {"type":"message_start","message":{"id":"msg_async_123","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}\n', + 'event: content_block_start\n', + 'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}\n', + 'event: content_block_delta\n', + 'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Async thinking..."}}\n', + 'event: content_block_stop\n', + 'data: {"type":"content_block_stop","index":0}\n', + 'event: content_block_start\n', + 'data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_async_01","name":"do_action","input":{}}}\n', + 'event: content_block_delta\n', + 'data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\\"action\\": \\"run\\"}"}}\n', + 'event: content_block_stop\n', + 'data: {"type":"content_block_stop","index":1}\n', + 'event: message_delta\n', + 'data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":15}}\n', + 'event: message_stop\n', + 'data: {"type":"message_stop"}\n', + ] + + class AsyncLineIterator: + def __init__(self, lines): + self._iter = iter(lines) + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self._iter) + except StopIteration: + raise StopAsyncIteration + + iterator = ModelResponseIterator( + streaming_response=AsyncLineIterator(sse_lines), + sync_stream=False, + json_mode=False, + ) + + chunks = [] + async for chunk in iterator: + chunks.append(chunk) + + assert all(isinstance(c, litellm.ModelResponseStream) for c in chunks) + tool_call_chunks = [ + c for c in chunks + if getattr(c.choices[0].delta, "tool_calls", None) is not None + ] + assert len(tool_call_chunks) == 2 + assert tool_call_chunks[0].choices[0].delta.tool_calls[0].function.name == "do_action" + + built = litellm.stream_chunk_builder(chunks=chunks) + assert built.choices[0].message.tool_calls[0].function.name == "do_action" + assert json.loads(built.choices[0].message.tool_calls[0].function.arguments) == {"action": "run"} + From a6082c93dc6d3ee8ef6f0089f2cc6bada349a291 Mon Sep 17 00:00:00 2001 From: elyar1124 <86278979+elyar1124@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:37:59 +0800 Subject: [PATCH 2/4] test(anthropic): trim redundant streaming test comments --- .../anthropic/chat/test_anthropic_chat_handler.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index 0ac0d2573ab..d365ad384e2 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -2571,12 +2571,6 @@ class TestRustChatCompletionsHook: def test_anthropic_streaming_thinking_text_tool_use_sync(): - """ - Regression test for Issue #36262: - Ensure Anthropic SSE stream containing thinking -> text -> tool_use emits only - ModelResponseStream instances (no dummy GenericStreamingChunk dicts), - preserves delta.tool_calls, and successfully builds via stream_chunk_builder. - """ sse_lines = [ b'event: message_start\n', b'data: {"type":"message_start","message":{"id":"msg_test_123","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}\n', @@ -2625,10 +2619,8 @@ def test_anthropic_streaming_thinking_text_tool_use_sync(): ) chunks = list(iterator) - # All emitted chunks must be ModelResponseStream instances assert all(isinstance(c, litellm.ModelResponseStream) for c in chunks) - # Tool call chunks must be present tool_call_chunks = [ c for c in chunks if getattr(c.choices[0].delta, "tool_calls", None) is not None @@ -2637,7 +2629,6 @@ def test_anthropic_streaming_thinking_text_tool_use_sync(): assert tool_call_chunks[0].choices[0].delta.tool_calls[0].function.name == "submit_result" assert tool_call_chunks[0].choices[0].delta.tool_calls[0].id == "toolu_01" - # stream_chunk_builder must reconstruct the message with tool calls built = litellm.stream_chunk_builder(chunks=chunks) assert built.choices[0].message.content == "I will now submit the result." assert built.choices[0].message.tool_calls is not None @@ -2649,10 +2640,6 @@ def test_anthropic_streaming_thinking_text_tool_use_sync(): @pytest.mark.asyncio async def test_anthropic_streaming_thinking_text_tool_use_async(): - """ - Regression test for Issue #36262 in async streaming (__anext__): - Ensure async iteration correctly yields all ModelResponseStream chunks and preserves tool_calls. - """ sse_lines = [ 'event: message_start\n', 'data: {"type":"message_start","message":{"id":"msg_async_123","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}\n', @@ -2708,4 +2695,3 @@ async def test_anthropic_streaming_thinking_text_tool_use_async(): built = litellm.stream_chunk_builder(chunks=chunks) assert built.choices[0].message.tool_calls[0].function.name == "do_action" assert json.loads(built.choices[0].message.tool_calls[0].function.arguments) == {"action": "run"} - From 1483a0185bde1db28bfa168c04f90b71caa85b78 Mon Sep 17 00:00:00 2001 From: elyar1124 <86278979+elyar1124@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:32:18 +0800 Subject: [PATCH 3/4] test(anthropic): avoid test append conflict with main --- .../chat/test_anthropic_chat_handler.py | 248 +++++++++--------- 1 file changed, 121 insertions(+), 127 deletions(-) diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index d365ad384e2..11d4d0b92f6 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -1844,6 +1844,127 @@ def test_streaming_code_execution_produces_code_interpreter_results(): ) +def test_anthropic_streaming_thinking_text_tool_use_sync(): + sse_lines = [ + b"event: message_start\n", + b'data: {"type":"message_start","message":{"id":"msg_test_123","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}\n', + b"\n", + b"event: content_block_start\n", + b'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}\n', + b"\n", + b"event: content_block_delta\n", + b'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Let me call submit_result"}}\n', + b"\n", + b"event: content_block_stop\n", + b'data: {"type":"content_block_stop","index":0}\n', + b"\n", + b"event: content_block_start\n", + b'data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}\n', + b"\n", + b"event: content_block_delta\n", + b'data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"I will now submit the result."}}\n', + b"\n", + b"event: content_block_stop\n", + b'data: {"type":"content_block_stop","index":1}\n', + b"\n", + b"event: content_block_start\n", + b'data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_01","name":"submit_result","input":{}}}\n', + b"\n", + b"event: content_block_delta\n", + b'data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\\"value\\":"}}\n', + b"\n", + b"event: content_block_delta\n", + b'data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"1}"}}\n', + b"\n", + b"event: content_block_stop\n", + b'data: {"type":"content_block_stop","index":2}\n', + b"\n", + b"event: message_delta\n", + b'data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":25}}\n', + b"\n", + b"event: message_stop\n", + b'data: {"type":"message_stop"}\n', + ] + + iterator = ModelResponseIterator( + streaming_response=iter(sse_lines), + sync_stream=True, + json_mode=False, + ) + chunks = list(iterator) + + assert all(isinstance(c, litellm.ModelResponseStream) for c in chunks) + + tool_call_chunks = [c for c in chunks if getattr(c.choices[0].delta, "tool_calls", None) is not None] + assert len(tool_call_chunks) == 3 + assert tool_call_chunks[0].choices[0].delta.tool_calls[0].function.name == "submit_result" + assert tool_call_chunks[0].choices[0].delta.tool_calls[0].id == "toolu_01" + + built = litellm.stream_chunk_builder(chunks=chunks) + assert built.choices[0].message.content == "I will now submit the result." + assert built.choices[0].message.tool_calls is not None + assert len(built.choices[0].message.tool_calls) == 1 + assert built.choices[0].message.tool_calls[0].function.name == "submit_result" + assert built.choices[0].message.tool_calls[0].function.arguments == '{"value":1}' + assert built.choices[0].finish_reason == "tool_calls" + + +@pytest.mark.asyncio +async def test_anthropic_streaming_thinking_text_tool_use_async(): + sse_lines = [ + "event: message_start\n", + 'data: {"type":"message_start","message":{"id":"msg_async_123","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}\n', + "event: content_block_start\n", + 'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}\n', + "event: content_block_delta\n", + 'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Async thinking..."}}\n', + "event: content_block_stop\n", + 'data: {"type":"content_block_stop","index":0}\n', + "event: content_block_start\n", + 'data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_async_01","name":"do_action","input":{}}}\n', + "event: content_block_delta\n", + 'data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\\"action\\": \\"run\\"}"}}\n', + "event: content_block_stop\n", + 'data: {"type":"content_block_stop","index":1}\n', + "event: message_delta\n", + 'data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":15}}\n', + "event: message_stop\n", + 'data: {"type":"message_stop"}\n', + ] + + class AsyncLineIterator: + def __init__(self, lines): + self._iter = iter(lines) + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self._iter) + except StopIteration: + raise StopAsyncIteration + + iterator = ModelResponseIterator( + streaming_response=AsyncLineIterator(sse_lines), + sync_stream=False, + json_mode=False, + ) + + chunks = [] + async for chunk in iterator: + chunks.append(chunk) + + assert all(isinstance(c, litellm.ModelResponseStream) for c in chunks) + tool_call_chunks = [c for c in chunks if getattr(c.choices[0].delta, "tool_calls", None) is not None] + assert len(tool_call_chunks) == 2 + assert tool_call_chunks[0].choices[0].delta.tool_calls[0].function.name == "do_action" + + built = litellm.stream_chunk_builder(chunks=chunks) + assert built.choices[0].message.tool_calls[0].function.name == "do_action" + assert json.loads(built.choices[0].message.tool_calls[0].function.arguments) == {"action": "run"} + + def test_streaming_multiple_code_executions_no_duplicates(): """ Test that multiple code executions in a single streaming response emit @@ -2568,130 +2689,3 @@ class TestRustChatCompletionsHook: "model": "m", "messages": [], } - - -def test_anthropic_streaming_thinking_text_tool_use_sync(): - sse_lines = [ - b'event: message_start\n', - b'data: {"type":"message_start","message":{"id":"msg_test_123","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}\n', - b'\n', - b'event: content_block_start\n', - b'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}\n', - b'\n', - b'event: content_block_delta\n', - b'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Let me call submit_result"}}\n', - b'\n', - b'event: content_block_stop\n', - b'data: {"type":"content_block_stop","index":0}\n', - b'\n', - b'event: content_block_start\n', - b'data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}\n', - b'\n', - b'event: content_block_delta\n', - b'data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"I will now submit the result."}}\n', - b'\n', - b'event: content_block_stop\n', - b'data: {"type":"content_block_stop","index":1}\n', - b'\n', - b'event: content_block_start\n', - b'data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_01","name":"submit_result","input":{}}}\n', - b'\n', - b'event: content_block_delta\n', - b'data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\\"value\\":"}}\n', - b'\n', - b'event: content_block_delta\n', - b'data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"1}"}}\n', - b'\n', - b'event: content_block_stop\n', - b'data: {"type":"content_block_stop","index":2}\n', - b'\n', - b'event: message_delta\n', - b'data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":25}}\n', - b'\n', - b'event: message_stop\n', - b'data: {"type":"message_stop"}\n', - ] - - iterator = ModelResponseIterator( - streaming_response=iter(sse_lines), - sync_stream=True, - json_mode=False, - ) - chunks = list(iterator) - - assert all(isinstance(c, litellm.ModelResponseStream) for c in chunks) - - tool_call_chunks = [ - c for c in chunks - if getattr(c.choices[0].delta, "tool_calls", None) is not None - ] - assert len(tool_call_chunks) == 3 - assert tool_call_chunks[0].choices[0].delta.tool_calls[0].function.name == "submit_result" - assert tool_call_chunks[0].choices[0].delta.tool_calls[0].id == "toolu_01" - - built = litellm.stream_chunk_builder(chunks=chunks) - assert built.choices[0].message.content == "I will now submit the result." - assert built.choices[0].message.tool_calls is not None - assert len(built.choices[0].message.tool_calls) == 1 - assert built.choices[0].message.tool_calls[0].function.name == "submit_result" - assert built.choices[0].message.tool_calls[0].function.arguments == '{"value":1}' - assert built.choices[0].finish_reason == "tool_calls" - - -@pytest.mark.asyncio -async def test_anthropic_streaming_thinking_text_tool_use_async(): - sse_lines = [ - 'event: message_start\n', - 'data: {"type":"message_start","message":{"id":"msg_async_123","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}\n', - 'event: content_block_start\n', - 'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}\n', - 'event: content_block_delta\n', - 'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Async thinking..."}}\n', - 'event: content_block_stop\n', - 'data: {"type":"content_block_stop","index":0}\n', - 'event: content_block_start\n', - 'data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_async_01","name":"do_action","input":{}}}\n', - 'event: content_block_delta\n', - 'data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\\"action\\": \\"run\\"}"}}\n', - 'event: content_block_stop\n', - 'data: {"type":"content_block_stop","index":1}\n', - 'event: message_delta\n', - 'data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":15}}\n', - 'event: message_stop\n', - 'data: {"type":"message_stop"}\n', - ] - - class AsyncLineIterator: - def __init__(self, lines): - self._iter = iter(lines) - - def __aiter__(self): - return self - - async def __anext__(self): - try: - return next(self._iter) - except StopIteration: - raise StopAsyncIteration - - iterator = ModelResponseIterator( - streaming_response=AsyncLineIterator(sse_lines), - sync_stream=False, - json_mode=False, - ) - - chunks = [] - async for chunk in iterator: - chunks.append(chunk) - - assert all(isinstance(c, litellm.ModelResponseStream) for c in chunks) - tool_call_chunks = [ - c for c in chunks - if getattr(c.choices[0].delta, "tool_calls", None) is not None - ] - assert len(tool_call_chunks) == 2 - assert tool_call_chunks[0].choices[0].delta.tool_calls[0].function.name == "do_action" - - built = litellm.stream_chunk_builder(chunks=chunks) - assert built.choices[0].message.tool_calls[0].function.name == "do_action" - assert json.loads(built.choices[0].message.tool_calls[0].function.arguments) == {"action": "run"} From 5731c58e060928aa5d218d1da31fd45b064ec83a Mon Sep 17 00:00:00 2001 From: elyar1124 <86278979+elyar1124@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:13 +0800 Subject: [PATCH 4/4] test(anthropic): isolate SSE regressions from renamed handler tests --- .../chat/test_anthropic_chat_handler.py | 121 ----------------- .../chat/test_anthropic_sse_framing.py | 127 ++++++++++++++++++ 2 files changed, 127 insertions(+), 121 deletions(-) create mode 100644 tests/unit/llms/anthropic/chat/test_anthropic_sse_framing.py diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index 11d4d0b92f6..bd750a47f63 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -1844,127 +1844,6 @@ def test_streaming_code_execution_produces_code_interpreter_results(): ) -def test_anthropic_streaming_thinking_text_tool_use_sync(): - sse_lines = [ - b"event: message_start\n", - b'data: {"type":"message_start","message":{"id":"msg_test_123","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}\n', - b"\n", - b"event: content_block_start\n", - b'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}\n', - b"\n", - b"event: content_block_delta\n", - b'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Let me call submit_result"}}\n', - b"\n", - b"event: content_block_stop\n", - b'data: {"type":"content_block_stop","index":0}\n', - b"\n", - b"event: content_block_start\n", - b'data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}\n', - b"\n", - b"event: content_block_delta\n", - b'data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"I will now submit the result."}}\n', - b"\n", - b"event: content_block_stop\n", - b'data: {"type":"content_block_stop","index":1}\n', - b"\n", - b"event: content_block_start\n", - b'data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_01","name":"submit_result","input":{}}}\n', - b"\n", - b"event: content_block_delta\n", - b'data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\\"value\\":"}}\n', - b"\n", - b"event: content_block_delta\n", - b'data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"1}"}}\n', - b"\n", - b"event: content_block_stop\n", - b'data: {"type":"content_block_stop","index":2}\n', - b"\n", - b"event: message_delta\n", - b'data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":25}}\n', - b"\n", - b"event: message_stop\n", - b'data: {"type":"message_stop"}\n', - ] - - iterator = ModelResponseIterator( - streaming_response=iter(sse_lines), - sync_stream=True, - json_mode=False, - ) - chunks = list(iterator) - - assert all(isinstance(c, litellm.ModelResponseStream) for c in chunks) - - tool_call_chunks = [c for c in chunks if getattr(c.choices[0].delta, "tool_calls", None) is not None] - assert len(tool_call_chunks) == 3 - assert tool_call_chunks[0].choices[0].delta.tool_calls[0].function.name == "submit_result" - assert tool_call_chunks[0].choices[0].delta.tool_calls[0].id == "toolu_01" - - built = litellm.stream_chunk_builder(chunks=chunks) - assert built.choices[0].message.content == "I will now submit the result." - assert built.choices[0].message.tool_calls is not None - assert len(built.choices[0].message.tool_calls) == 1 - assert built.choices[0].message.tool_calls[0].function.name == "submit_result" - assert built.choices[0].message.tool_calls[0].function.arguments == '{"value":1}' - assert built.choices[0].finish_reason == "tool_calls" - - -@pytest.mark.asyncio -async def test_anthropic_streaming_thinking_text_tool_use_async(): - sse_lines = [ - "event: message_start\n", - 'data: {"type":"message_start","message":{"id":"msg_async_123","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}\n', - "event: content_block_start\n", - 'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}\n', - "event: content_block_delta\n", - 'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Async thinking..."}}\n', - "event: content_block_stop\n", - 'data: {"type":"content_block_stop","index":0}\n', - "event: content_block_start\n", - 'data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_async_01","name":"do_action","input":{}}}\n', - "event: content_block_delta\n", - 'data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\\"action\\": \\"run\\"}"}}\n', - "event: content_block_stop\n", - 'data: {"type":"content_block_stop","index":1}\n', - "event: message_delta\n", - 'data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":15}}\n', - "event: message_stop\n", - 'data: {"type":"message_stop"}\n', - ] - - class AsyncLineIterator: - def __init__(self, lines): - self._iter = iter(lines) - - def __aiter__(self): - return self - - async def __anext__(self): - try: - return next(self._iter) - except StopIteration: - raise StopAsyncIteration - - iterator = ModelResponseIterator( - streaming_response=AsyncLineIterator(sse_lines), - sync_stream=False, - json_mode=False, - ) - - chunks = [] - async for chunk in iterator: - chunks.append(chunk) - - assert all(isinstance(c, litellm.ModelResponseStream) for c in chunks) - tool_call_chunks = [c for c in chunks if getattr(c.choices[0].delta, "tool_calls", None) is not None] - assert len(tool_call_chunks) == 2 - assert tool_call_chunks[0].choices[0].delta.tool_calls[0].function.name == "do_action" - - built = litellm.stream_chunk_builder(chunks=chunks) - assert built.choices[0].message.tool_calls[0].function.name == "do_action" - assert json.loads(built.choices[0].message.tool_calls[0].function.arguments) == {"action": "run"} - - def test_streaming_multiple_code_executions_no_duplicates(): """ Test that multiple code executions in a single streaming response emit diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_sse_framing.py b/tests/unit/llms/anthropic/chat/test_anthropic_sse_framing.py new file mode 100644 index 00000000000..fc97edf246e --- /dev/null +++ b/tests/unit/llms/anthropic/chat/test_anthropic_sse_framing.py @@ -0,0 +1,127 @@ +import json + +import pytest + +import litellm +from litellm.llms.anthropic.chat.handler import ModelResponseIterator + + +def test_anthropic_streaming_thinking_text_tool_use_sync(): + sse_lines = [ + b"event: message_start\n", + b'data: {"type":"message_start","message":{"id":"msg_test_123","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}\n', + b"\n", + b"event: content_block_start\n", + b'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}\n', + b"\n", + b"event: content_block_delta\n", + b'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Let me call submit_result"}}\n', + b"\n", + b"event: content_block_stop\n", + b'data: {"type":"content_block_stop","index":0}\n', + b"\n", + b"event: content_block_start\n", + b'data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}\n', + b"\n", + b"event: content_block_delta\n", + b'data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"I will now submit the result."}}\n', + b"\n", + b"event: content_block_stop\n", + b'data: {"type":"content_block_stop","index":1}\n', + b"\n", + b"event: content_block_start\n", + b'data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_01","name":"submit_result","input":{}}}\n', + b"\n", + b"event: content_block_delta\n", + b'data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\\"value\\":"}}\n', + b"\n", + b"event: content_block_delta\n", + b'data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"1}"}}\n', + b"\n", + b"event: content_block_stop\n", + b'data: {"type":"content_block_stop","index":2}\n', + b"\n", + b"event: message_delta\n", + b'data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":25}}\n', + b"\n", + b"event: message_stop\n", + b'data: {"type":"message_stop"}\n', + ] + + iterator = ModelResponseIterator( + streaming_response=iter(sse_lines), + sync_stream=True, + json_mode=False, + ) + chunks = list(iterator) + + assert all(isinstance(c, litellm.ModelResponseStream) for c in chunks) + + tool_call_chunks = [c for c in chunks if getattr(c.choices[0].delta, "tool_calls", None) is not None] + assert len(tool_call_chunks) == 3 + assert tool_call_chunks[0].choices[0].delta.tool_calls[0].function.name == "submit_result" + assert tool_call_chunks[0].choices[0].delta.tool_calls[0].id == "toolu_01" + + built = litellm.stream_chunk_builder(chunks=chunks) + assert built.choices[0].message.content == "I will now submit the result." + assert built.choices[0].message.tool_calls is not None + assert len(built.choices[0].message.tool_calls) == 1 + assert built.choices[0].message.tool_calls[0].function.name == "submit_result" + assert built.choices[0].message.tool_calls[0].function.arguments == '{"value":1}' + assert built.choices[0].finish_reason == "tool_calls" + + +@pytest.mark.asyncio +async def test_anthropic_streaming_thinking_text_tool_use_async(): + sse_lines = [ + "event: message_start\n", + 'data: {"type":"message_start","message":{"id":"msg_async_123","type":"message","role":"assistant","model":"claude-3-7-sonnet-20250219","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}\n', + "event: content_block_start\n", + 'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}\n', + "event: content_block_delta\n", + 'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Async thinking..."}}\n', + "event: content_block_stop\n", + 'data: {"type":"content_block_stop","index":0}\n', + "event: content_block_start\n", + 'data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_async_01","name":"do_action","input":{}}}\n', + "event: content_block_delta\n", + 'data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\\"action\\": \\"run\\"}"}}\n', + "event: content_block_stop\n", + 'data: {"type":"content_block_stop","index":1}\n', + "event: message_delta\n", + 'data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":15}}\n', + "event: message_stop\n", + 'data: {"type":"message_stop"}\n', + ] + + class AsyncLineIterator: + def __init__(self, lines): + self._iter = iter(lines) + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self._iter) + except StopIteration: + raise StopAsyncIteration + + iterator = ModelResponseIterator( + streaming_response=AsyncLineIterator(sse_lines), + sync_stream=False, + json_mode=False, + ) + + chunks = [] + async for chunk in iterator: + chunks.append(chunk) + + assert all(isinstance(c, litellm.ModelResponseStream) for c in chunks) + tool_call_chunks = [c for c in chunks if getattr(c.choices[0].delta, "tool_calls", None) is not None] + assert len(tool_call_chunks) == 2 + assert tool_call_chunks[0].choices[0].delta.tool_calls[0].function.name == "do_action" + + built = litellm.stream_chunk_builder(chunks=chunks) + assert built.choices[0].message.tool_calls[0].function.name == "do_action" + assert json.loads(built.choices[0].message.tool_calls[0].function.arguments) == {"action": "run"}