test: add non-bash skip test and mock end-to-end streaming integration test

- test_non_bash_tool_result_skipped: verifies text_editor results produce
  zero code_interpreter_call items
- test_end_to_end_streaming_chunks_to_code_interpreter_output: exercises
  full path from Anthropic SSE chunks through ModelResponseIterator,
  stream_chunk_builder, and _extract_tool_result_output_items without
  a live server
This commit is contained in:
Andrzej Pomirski 2026-03-18 11:15:15 +01:00
parent 8f60117228
commit d10007cef4
2 changed files with 172 additions and 2 deletions

View file

@ -1513,3 +1513,71 @@ def test_empty_output_produces_null_outputs():
assert (
code_results[0].outputs is None
), f"Expected outputs=None for empty execution, got {code_results[0].outputs}"
def test_non_bash_tool_result_skipped():
"""
Tool result types other than bash_code_execution_tool_result (e.g.
text_editor_code_execution_tool_result) should be skipped and NOT
produce code_interpreter_call items.
"""
chunks = [
{
"type": "message_start",
"message": {
"id": "msg_01XYZ",
"type": "message",
"role": "assistant",
"content": [],
"usage": {"input_tokens": 100, "output_tokens": 1},
},
},
{
"type": "content_block_start",
"index": 0,
"content_block": {
"type": "server_tool_use",
"id": "srvtoolu_01AAA",
"name": "text_editor",
"input": {"command": "view", "path": "/tmp/test.py"},
},
},
{"type": "content_block_stop", "index": 0},
# text_editor result — should NOT become a code_interpreter_call
{
"type": "content_block_start",
"index": 1,
"content_block": {
"type": "text_editor_code_execution_tool_result",
"tool_use_id": "srvtoolu_01AAA",
"content": [
{"type": "text", "text": "file contents here"},
],
},
},
{"type": "content_block_stop", "index": 1},
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn"},
"usage": {"output_tokens": 50},
},
]
iterator = ModelResponseIterator(None, sync_stream=True)
code_results = None
for chunk in chunks:
parsed = iterator.chunk_parser(chunk)
psf = None
if parsed.choices and parsed.choices[0].delta:
psf = getattr(parsed.choices[0].delta, "provider_specific_fields", None)
if psf and "code_interpreter_results" in psf:
code_results = psf["code_interpreter_results"]
# code_interpreter_results should be emitted but empty (no bash results)
assert (
code_results is not None
), "Expected code_interpreter_results key to be emitted"
assert (
len(code_results) == 0
), f"Expected 0 code_interpreter_results for text_editor result, got {len(code_results)}"

View file

@ -1,10 +1,13 @@
"""
Tests for the Responses API _extract_tool_result_output_items path
and the non-streaming _hidden_params propagation of code_interpreter_results.
Tests for the Responses API _extract_tool_result_output_items path,
the non-streaming _hidden_params propagation of code_interpreter_results,
and mock end-to-end streaming integration.
"""
from unittest.mock import MagicMock
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
from litellm.main import stream_chunk_builder
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
@ -161,3 +164,102 @@ def test_in_place_substitution_preserves_ordering():
assert responses_output[2].call_id == "srvtoolu_01BBB"
assert responses_output[3].type == "code_interpreter_call"
assert responses_output[3].id == "srvtoolu_01CCC"
def test_end_to_end_streaming_chunks_to_code_interpreter_output():
"""
Mock end-to-end test: Anthropic SSE chunks ModelResponseIterator
stream_chunk_builder _extract_tool_result_output_items final output
with code_interpreter_call items replacing function_call items.
This exercises the full streaming data flow without a live server.
"""
# Realistic Anthropic streaming chunks for a single code execution
raw_chunks = [
{
"type": "message_start",
"message": {
"id": "msg_01XYZ",
"type": "message",
"role": "assistant",
"content": [],
"usage": {"input_tokens": 100, "output_tokens": 1},
},
},
{
"type": "content_block_start",
"index": 0,
"content_block": {
"type": "server_tool_use",
"id": "srvtoolu_01AAA",
"name": "bash_code_execution",
"input": {},
},
},
{
"type": "content_block_delta",
"index": 0,
"delta": {
"type": "input_json_delta",
"partial_json": '{"command": "echo e2e_test"}',
},
},
{"type": "content_block_stop", "index": 0},
{
"type": "content_block_start",
"index": 1,
"content_block": {
"type": "bash_code_execution_tool_result",
"tool_use_id": "srvtoolu_01AAA",
"content": {
"type": "bash_code_execution_result",
"stdout": "e2e_test\n",
"stderr": "",
"return_code": 0,
},
},
},
{"type": "content_block_stop", "index": 1},
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn"},
"usage": {"output_tokens": 50},
},
]
# Step 1: Parse chunks through ModelResponseIterator (Anthropic handler)
iterator = ModelResponseIterator(None, sync_stream=True)
parsed_chunks = []
for chunk in raw_chunks:
parsed = iterator.chunk_parser(chunk)
d = parsed.model_dump()
# In production, CustomStreamWrapper sets the model on each chunk;
# stream_chunk_builder requires it.
d["model"] = "claude-sonnet-4-20250514"
parsed_chunks.append(d)
# Step 2: Assemble via stream_chunk_builder (simulates end-of-stream)
assembled = stream_chunk_builder(chunks=parsed_chunks)
assert assembled is not None
# Verify stream_chunk_builder picked up code_interpreter_results via last-value-wins
psf = assembled.choices[0].message.provider_specific_fields
assert psf is not None
assert "code_interpreter_results" in psf
code_results = psf["code_interpreter_results"]
assert len(code_results) == 1
# After model_dump + stream_chunk_builder, results are plain dicts
assert code_results[0]["id"] == "srvtoolu_01AAA"
assert code_results[0]["code"] == "echo e2e_test"
# Step 3: Extract via _extract_tool_result_output_items (Responses API layer)
tool_result_items = (
LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(assembled)
)
assert len(tool_result_items) == 1
item = tool_result_items[0]
# Items are dicts after the model_dump path
assert item["type"] == "code_interpreter_call"
assert item["id"] == "srvtoolu_01AAA"
assert item["code"] == "echo e2e_test"
assert item["outputs"][0]["logs"] == "e2e_test\n"