mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-06 08:16:43 +00:00
test: add non-bash skip test and mock end-to-end streaming integration test
- test_non_bash_tool_result_skipped: verifies text_editor results produce zero code_interpreter_call items - test_end_to_end_streaming_chunks_to_code_interpreter_output: exercises full path from Anthropic SSE chunks through ModelResponseIterator, stream_chunk_builder, and _extract_tool_result_output_items without a live server
This commit is contained in:
parent
8f60117228
commit
d10007cef4
2 changed files with 172 additions and 2 deletions
|
|
@ -1513,3 +1513,71 @@ def test_empty_output_produces_null_outputs():
|
|||
assert (
|
||||
code_results[0].outputs is None
|
||||
), f"Expected outputs=None for empty execution, got {code_results[0].outputs}"
|
||||
|
||||
|
||||
def test_non_bash_tool_result_skipped():
|
||||
"""
|
||||
Tool result types other than bash_code_execution_tool_result (e.g.
|
||||
text_editor_code_execution_tool_result) should be skipped and NOT
|
||||
produce code_interpreter_call items.
|
||||
"""
|
||||
chunks = [
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_01XYZ",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [],
|
||||
"usage": {"input_tokens": 100, "output_tokens": 1},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 0,
|
||||
"content_block": {
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_01AAA",
|
||||
"name": "text_editor",
|
||||
"input": {"command": "view", "path": "/tmp/test.py"},
|
||||
},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
# text_editor result — should NOT become a code_interpreter_call
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 1,
|
||||
"content_block": {
|
||||
"type": "text_editor_code_execution_tool_result",
|
||||
"tool_use_id": "srvtoolu_01AAA",
|
||||
"content": [
|
||||
{"type": "text", "text": "file contents here"},
|
||||
],
|
||||
},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 1},
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "end_turn"},
|
||||
"usage": {"output_tokens": 50},
|
||||
},
|
||||
]
|
||||
|
||||
iterator = ModelResponseIterator(None, sync_stream=True)
|
||||
|
||||
code_results = None
|
||||
for chunk in chunks:
|
||||
parsed = iterator.chunk_parser(chunk)
|
||||
psf = None
|
||||
if parsed.choices and parsed.choices[0].delta:
|
||||
psf = getattr(parsed.choices[0].delta, "provider_specific_fields", None)
|
||||
if psf and "code_interpreter_results" in psf:
|
||||
code_results = psf["code_interpreter_results"]
|
||||
|
||||
# code_interpreter_results should be emitted but empty (no bash results)
|
||||
assert (
|
||||
code_results is not None
|
||||
), "Expected code_interpreter_results key to be emitted"
|
||||
assert (
|
||||
len(code_results) == 0
|
||||
), f"Expected 0 code_interpreter_results for text_editor result, got {len(code_results)}"
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
"""
|
||||
Tests for the Responses API _extract_tool_result_output_items path
|
||||
and the non-streaming _hidden_params propagation of code_interpreter_results.
|
||||
Tests for the Responses API _extract_tool_result_output_items path,
|
||||
the non-streaming _hidden_params propagation of code_interpreter_results,
|
||||
and mock end-to-end streaming integration.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
|
||||
from litellm.main import stream_chunk_builder
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
)
|
||||
|
|
@ -161,3 +164,102 @@ def test_in_place_substitution_preserves_ordering():
|
|||
assert responses_output[2].call_id == "srvtoolu_01BBB"
|
||||
assert responses_output[3].type == "code_interpreter_call"
|
||||
assert responses_output[3].id == "srvtoolu_01CCC"
|
||||
|
||||
|
||||
def test_end_to_end_streaming_chunks_to_code_interpreter_output():
|
||||
"""
|
||||
Mock end-to-end test: Anthropic SSE chunks → ModelResponseIterator →
|
||||
stream_chunk_builder → _extract_tool_result_output_items → final output
|
||||
with code_interpreter_call items replacing function_call items.
|
||||
|
||||
This exercises the full streaming data flow without a live server.
|
||||
"""
|
||||
# Realistic Anthropic streaming chunks for a single code execution
|
||||
raw_chunks = [
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_01XYZ",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [],
|
||||
"usage": {"input_tokens": 100, "output_tokens": 1},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 0,
|
||||
"content_block": {
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_01AAA",
|
||||
"name": "bash_code_execution",
|
||||
"input": {},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"type": "input_json_delta",
|
||||
"partial_json": '{"command": "echo e2e_test"}',
|
||||
},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 1,
|
||||
"content_block": {
|
||||
"type": "bash_code_execution_tool_result",
|
||||
"tool_use_id": "srvtoolu_01AAA",
|
||||
"content": {
|
||||
"type": "bash_code_execution_result",
|
||||
"stdout": "e2e_test\n",
|
||||
"stderr": "",
|
||||
"return_code": 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 1},
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "end_turn"},
|
||||
"usage": {"output_tokens": 50},
|
||||
},
|
||||
]
|
||||
|
||||
# Step 1: Parse chunks through ModelResponseIterator (Anthropic handler)
|
||||
iterator = ModelResponseIterator(None, sync_stream=True)
|
||||
parsed_chunks = []
|
||||
for chunk in raw_chunks:
|
||||
parsed = iterator.chunk_parser(chunk)
|
||||
d = parsed.model_dump()
|
||||
# In production, CustomStreamWrapper sets the model on each chunk;
|
||||
# stream_chunk_builder requires it.
|
||||
d["model"] = "claude-sonnet-4-20250514"
|
||||
parsed_chunks.append(d)
|
||||
|
||||
# Step 2: Assemble via stream_chunk_builder (simulates end-of-stream)
|
||||
assembled = stream_chunk_builder(chunks=parsed_chunks)
|
||||
assert assembled is not None
|
||||
|
||||
# Verify stream_chunk_builder picked up code_interpreter_results via last-value-wins
|
||||
psf = assembled.choices[0].message.provider_specific_fields
|
||||
assert psf is not None
|
||||
assert "code_interpreter_results" in psf
|
||||
code_results = psf["code_interpreter_results"]
|
||||
assert len(code_results) == 1
|
||||
# After model_dump + stream_chunk_builder, results are plain dicts
|
||||
assert code_results[0]["id"] == "srvtoolu_01AAA"
|
||||
assert code_results[0]["code"] == "echo e2e_test"
|
||||
|
||||
# Step 3: Extract via _extract_tool_result_output_items (Responses API layer)
|
||||
tool_result_items = (
|
||||
LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(assembled)
|
||||
)
|
||||
assert len(tool_result_items) == 1
|
||||
item = tool_result_items[0]
|
||||
# Items are dicts after the model_dump path
|
||||
assert item["type"] == "code_interpreter_call"
|
||||
assert item["id"] == "srvtoolu_01AAA"
|
||||
assert item["code"] == "echo e2e_test"
|
||||
assert item["outputs"][0]["logs"] == "e2e_test\n"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue