mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
tests(claude_code): bugbot — raise MIN_STREAM_EVENTS for tool-use streaming on remaining 4 providers
Per Bugbot, MIN_STREAM_EVENTS=4 is below the buffered-proxy baseline for
tool-use streaming: even a buffered wire produces ~5 stream-json records
(system + assistant w/ tool_use + user tool_result + assistant text +
result), so the < 4 check would silently pass for a regression that
collapses the upstream stream. Bring azure / bedrock_invoke /
bedrock_converse / vertex_ai in line with test_anthropic.py (MIN=8,
already raised in a253bd940b) so every provider catches the regression
this row is designed to detect.
Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com>
This commit is contained in:
parent
0c9026bb20
commit
0bfab91ec9
4 changed files with 48 additions and 4 deletions
|
|
@ -50,7 +50,18 @@ TOOL_USE_ARGS = [
|
|||
"dontAsk",
|
||||
]
|
||||
|
||||
MIN_STREAM_EVENTS = 4
|
||||
# Floor on the number of stream-json records we expect to see for a
|
||||
# tool-use turn. A buffered (non-streamed) wire for this multi-turn
|
||||
# flow collapses to roughly: one `system` init + one `assistant` with
|
||||
# the `tool_use` block + a `user` tool_result + one `assistant` final
|
||||
# text + one `result`, i.e. ~5 records (the CLI executes the tool
|
||||
# locally and sends the result back, producing a second model turn
|
||||
# even on a fully buffered proxy). Real fine-grained streaming
|
||||
# produces many more (incremental input_json_delta events,
|
||||
# intermediate assistant deltas, etc., typically 15+). We pick a
|
||||
# floor comfortably above the buffered case so the assertion catches
|
||||
# the regression without being flaky on short responses.
|
||||
MIN_STREAM_EVENTS = 8
|
||||
|
||||
|
||||
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
|
||||
|
|
|
|||
|
|
@ -56,7 +56,18 @@ TOOL_USE_ARGS = [
|
|||
"dontAsk",
|
||||
]
|
||||
|
||||
MIN_STREAM_EVENTS = 4
|
||||
# Floor on the number of stream-json records we expect to see for a
|
||||
# tool-use turn. A buffered (non-streamed) wire for this multi-turn
|
||||
# flow collapses to roughly: one `system` init + one `assistant` with
|
||||
# the `tool_use` block + a `user` tool_result + one `assistant` final
|
||||
# text + one `result`, i.e. ~5 records (the CLI executes the tool
|
||||
# locally and sends the result back, producing a second model turn
|
||||
# even on a fully buffered proxy). Real fine-grained streaming
|
||||
# produces many more (incremental input_json_delta events,
|
||||
# intermediate assistant deltas, etc., typically 15+). We pick a
|
||||
# floor comfortably above the buffered case so the assertion catches
|
||||
# the regression without being flaky on short responses.
|
||||
MIN_STREAM_EVENTS = 8
|
||||
|
||||
|
||||
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
|
||||
|
|
|
|||
|
|
@ -54,7 +54,18 @@ TOOL_USE_ARGS = [
|
|||
"dontAsk",
|
||||
]
|
||||
|
||||
MIN_STREAM_EVENTS = 4
|
||||
# Floor on the number of stream-json records we expect to see for a
|
||||
# tool-use turn. A buffered (non-streamed) wire for this multi-turn
|
||||
# flow collapses to roughly: one `system` init + one `assistant` with
|
||||
# the `tool_use` block + a `user` tool_result + one `assistant` final
|
||||
# text + one `result`, i.e. ~5 records (the CLI executes the tool
|
||||
# locally and sends the result back, producing a second model turn
|
||||
# even on a fully buffered proxy). Real fine-grained streaming
|
||||
# produces many more (incremental input_json_delta events,
|
||||
# intermediate assistant deltas, etc., typically 15+). We pick a
|
||||
# floor comfortably above the buffered case so the assertion catches
|
||||
# the regression without being flaky on short responses.
|
||||
MIN_STREAM_EVENTS = 8
|
||||
|
||||
|
||||
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
|
||||
|
|
|
|||
|
|
@ -53,7 +53,18 @@ TOOL_USE_ARGS = [
|
|||
"dontAsk",
|
||||
]
|
||||
|
||||
MIN_STREAM_EVENTS = 4
|
||||
# Floor on the number of stream-json records we expect to see for a
|
||||
# tool-use turn. A buffered (non-streamed) wire for this multi-turn
|
||||
# flow collapses to roughly: one `system` init + one `assistant` with
|
||||
# the `tool_use` block + a `user` tool_result + one `assistant` final
|
||||
# text + one `result`, i.e. ~5 records (the CLI executes the tool
|
||||
# locally and sends the result back, producing a second model turn
|
||||
# even on a fully buffered proxy). Real fine-grained streaming
|
||||
# produces many more (incremental input_json_delta events,
|
||||
# intermediate assistant deltas, etc., typically 15+). We pick a
|
||||
# floor comfortably above the buffered case so the assertion catches
|
||||
# the regression without being flaky on short responses.
|
||||
MIN_STREAM_EVENTS = 8
|
||||
|
||||
|
||||
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue