mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(chatgpt): document incremental stream state mutability
Keep committed files LF-normalized when publishing via API.
This commit is contained in:
parent
e9a77e77f4
commit
0803a83aee
2 changed files with 368 additions and 368 deletions
|
|
@ -1,118 +1,118 @@
|
|||
"""
|
||||
Streaming utilities for ChatGPT provider.
|
||||
|
||||
Normalizes non-spec-compliant tool_call chunks from the ChatGPT backend API.
|
||||
"""
|
||||
|
||||
from collections.abc import Awaitable
|
||||
from typing import Final, Protocol
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionDeltaCustomToolCall,
|
||||
ChatCompletionDeltaToolCall,
|
||||
Delta,
|
||||
ModelResponseStream,
|
||||
)
|
||||
|
||||
|
||||
class ChatGPTChunkStream(Protocol):
|
||||
"""A ChatGPT chunk source driven either synchronously or asynchronously."""
|
||||
|
||||
def __next__(self) -> ModelResponseStream: ...
|
||||
|
||||
def __anext__(self) -> Awaitable[ModelResponseStream]: ...
|
||||
|
||||
|
||||
_DELEGATED_ATTRIBUTE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _first_choice_delta(chunk: ModelResponseStream) -> Delta | None:
|
||||
return chunk.choices[0].delta
|
||||
|
||||
|
||||
class ChatGPTToolCallNormalizer:
|
||||
"""
|
||||
Wraps a streaming response and fixes tool_call index/dedup issues.
|
||||
|
||||
The ChatGPT backend API (chatgpt.com/backend-api) can send tool_call chunks
|
||||
with:
|
||||
1. `index` values that are sparse or repeated across parallel tool calls
|
||||
2. `id` and `name` repeated in "closing" chunks that shouldn't exist
|
||||
|
||||
This wrapper maps unique source indices to sequential output indices and
|
||||
uses the last identified call when repeated indices make a continuation
|
||||
ambiguous.
|
||||
"""
|
||||
|
||||
def __init__(self, stream: ChatGPTChunkStream):
|
||||
self._stream: Final = stream
|
||||
self._seen_ids: dict[str, int] = {} # tool_call_id -> assigned_index
|
||||
self._next_index: int = 0
|
||||
self._source_index_to_output_index: dict[int, int | None] = {}
|
||||
self._last_id: str | None = None # tracks which tool call the next delta belongs to
|
||||
|
||||
def __getattr__(self, name: str) -> object:
|
||||
return _DELEGATED_ATTRIBUTE.validate_python(getattr(self._stream, name))
|
||||
|
||||
def __iter__(self) -> "ChatGPTToolCallNormalizer":
|
||||
return self
|
||||
|
||||
def __aiter__(self) -> "ChatGPTToolCallNormalizer":
|
||||
return self
|
||||
|
||||
def __next__(self) -> ModelResponseStream:
|
||||
while True:
|
||||
chunk = next(self._stream)
|
||||
result = self._normalize(chunk)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
async def __anext__(self) -> ModelResponseStream:
|
||||
while True:
|
||||
chunk = await self._stream.__anext__()
|
||||
result = self._normalize(chunk)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
def _normalize(self, chunk: ModelResponseStream) -> ModelResponseStream | None:
|
||||
"""Fix tool_calls in the chunk. Returns None to skip duplicate chunks."""
|
||||
if not chunk.choices:
|
||||
return chunk
|
||||
|
||||
delta: Final = _first_choice_delta(chunk)
|
||||
if delta is None or not delta.tool_calls:
|
||||
return chunk
|
||||
|
||||
normalized: Final[list[ChatCompletionDeltaToolCall | ChatCompletionDeltaCustomToolCall]] = []
|
||||
for tc in delta.tool_calls:
|
||||
if tc.id and tc.id not in self._seen_ids:
|
||||
# New tool call — assign correct index
|
||||
if tc.index in self._source_index_to_output_index:
|
||||
# Preserve the last-id fallback for repeated source indices.
|
||||
self._source_index_to_output_index[tc.index] = None
|
||||
else:
|
||||
self._source_index_to_output_index[tc.index] = self._next_index
|
||||
self._seen_ids[tc.id] = self._next_index
|
||||
tc.index = self._next_index
|
||||
self._last_id = tc.id
|
||||
self._next_index += 1
|
||||
normalized.append(tc)
|
||||
elif tc.id and tc.id in self._seen_ids:
|
||||
# Duplicate "closing" chunk — skip it
|
||||
continue
|
||||
else:
|
||||
# Continuation delta (id=None) — use its source index when unambiguous
|
||||
normalized_index: Final = self._source_index_to_output_index.get(tc.index)
|
||||
if normalized_index is not None:
|
||||
tc.index = normalized_index
|
||||
elif self._last_id:
|
||||
tc.index = self._seen_ids[self._last_id]
|
||||
normalized.append(tc)
|
||||
|
||||
if not normalized:
|
||||
return None # all tool_calls were duplicates, skip chunk
|
||||
|
||||
delta.tool_calls = normalized
|
||||
return chunk
|
||||
"""
|
||||
Streaming utilities for ChatGPT provider.
|
||||
|
||||
Normalizes non-spec-compliant tool_call chunks from the ChatGPT backend API.
|
||||
"""
|
||||
|
||||
from collections.abc import Awaitable
|
||||
from typing import Final, Protocol
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionDeltaCustomToolCall,
|
||||
ChatCompletionDeltaToolCall,
|
||||
Delta,
|
||||
ModelResponseStream,
|
||||
)
|
||||
|
||||
|
||||
class ChatGPTChunkStream(Protocol):
|
||||
"""A ChatGPT chunk source driven either synchronously or asynchronously."""
|
||||
|
||||
def __next__(self) -> ModelResponseStream: ...
|
||||
|
||||
def __anext__(self) -> Awaitable[ModelResponseStream]: ...
|
||||
|
||||
|
||||
_DELEGATED_ATTRIBUTE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _first_choice_delta(chunk: ModelResponseStream) -> Delta | None:
|
||||
return chunk.choices[0].delta
|
||||
|
||||
|
||||
class ChatGPTToolCallNormalizer:
|
||||
"""
|
||||
Wraps a streaming response and fixes tool_call index/dedup issues.
|
||||
|
||||
The ChatGPT backend API (chatgpt.com/backend-api) can send tool_call chunks
|
||||
with:
|
||||
1. `index` values that are sparse or repeated across parallel tool calls
|
||||
2. `id` and `name` repeated in "closing" chunks that shouldn't exist
|
||||
|
||||
This wrapper maps unique source indices to sequential output indices and
|
||||
uses the last identified call when repeated indices make a continuation
|
||||
ambiguous.
|
||||
"""
|
||||
|
||||
def __init__(self, stream: ChatGPTChunkStream):
|
||||
self._stream: Final = stream
|
||||
self._seen_ids: dict[str, int] = {} # tool_call_id -> assigned_index
|
||||
self._next_index: int = 0
|
||||
self._source_index_to_output_index: dict[int, int | None] = {} # mutable-ok: incremental per-stream state
|
||||
self._last_id: str | None = None # tracks which tool call the next delta belongs to
|
||||
|
||||
def __getattr__(self, name: str) -> object:
|
||||
return _DELEGATED_ATTRIBUTE.validate_python(getattr(self._stream, name))
|
||||
|
||||
def __iter__(self) -> "ChatGPTToolCallNormalizer":
|
||||
return self
|
||||
|
||||
def __aiter__(self) -> "ChatGPTToolCallNormalizer":
|
||||
return self
|
||||
|
||||
def __next__(self) -> ModelResponseStream:
|
||||
while True:
|
||||
chunk = next(self._stream)
|
||||
result = self._normalize(chunk)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
async def __anext__(self) -> ModelResponseStream:
|
||||
while True:
|
||||
chunk = await self._stream.__anext__()
|
||||
result = self._normalize(chunk)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
def _normalize(self, chunk: ModelResponseStream) -> ModelResponseStream | None:
|
||||
"""Fix tool_calls in the chunk. Returns None to skip duplicate chunks."""
|
||||
if not chunk.choices:
|
||||
return chunk
|
||||
|
||||
delta: Final = _first_choice_delta(chunk)
|
||||
if delta is None or not delta.tool_calls:
|
||||
return chunk
|
||||
|
||||
normalized: Final[list[ChatCompletionDeltaToolCall | ChatCompletionDeltaCustomToolCall]] = []
|
||||
for tc in delta.tool_calls:
|
||||
if tc.id and tc.id not in self._seen_ids:
|
||||
# New tool call — assign correct index
|
||||
if tc.index in self._source_index_to_output_index:
|
||||
# Preserve the last-id fallback for repeated source indices.
|
||||
self._source_index_to_output_index[tc.index] = None
|
||||
else:
|
||||
self._source_index_to_output_index[tc.index] = self._next_index
|
||||
self._seen_ids[tc.id] = self._next_index
|
||||
tc.index = self._next_index
|
||||
self._last_id = tc.id
|
||||
self._next_index += 1
|
||||
normalized.append(tc)
|
||||
elif tc.id and tc.id in self._seen_ids:
|
||||
# Duplicate "closing" chunk — skip it
|
||||
continue
|
||||
else:
|
||||
# Continuation delta (id=None) — use its source index when unambiguous
|
||||
normalized_index: Final = self._source_index_to_output_index.get(tc.index)
|
||||
if normalized_index is not None:
|
||||
tc.index = normalized_index
|
||||
elif self._last_id:
|
||||
tc.index = self._seen_ids[self._last_id]
|
||||
normalized.append(tc)
|
||||
|
||||
if not normalized:
|
||||
return None # all tool_calls were duplicates, skip chunk
|
||||
|
||||
delta.tool_calls = normalized
|
||||
return chunk
|
||||
|
|
|
|||
|
|
@ -1,250 +1,250 @@
|
|||
"""
|
||||
Tests for ChatGPTToolCallNormalizer.
|
||||
|
||||
Verifies that non-spec-compliant tool_call chunks from the ChatGPT backend API
|
||||
are normalized to match the OpenAI streaming spec:
|
||||
- Correct index assignment for parallel tool calls
|
||||
- Deduplication of "closing" chunks with repeated id/name
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.chatgpt.chat.streaming_utils import ChatGPTToolCallNormalizer
|
||||
from litellm.llms.chatgpt.chat.transformation import ChatGPTConfig
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionDeltaToolCall,
|
||||
Delta,
|
||||
Function,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
)
|
||||
|
||||
|
||||
def _make_chunk(tool_calls=None, content=None):
|
||||
"""Helper to build a ModelResponseStream chunk with tool_calls on the delta."""
|
||||
delta = Delta(
|
||||
content=content,
|
||||
role="assistant",
|
||||
tool_calls=tool_calls,
|
||||
)
|
||||
choice = StreamingChoices(delta=delta, index=0)
|
||||
return ModelResponseStream(choices=[choice])
|
||||
|
||||
|
||||
def _make_tc(index=0, id=None, name=None, arguments=None):
|
||||
"""Helper to build a ChatCompletionDeltaToolCall."""
|
||||
func = Function(name=name, arguments=arguments)
|
||||
return ChatCompletionDeltaToolCall(
|
||||
index=index,
|
||||
id=id,
|
||||
function=func,
|
||||
type="function" if id else None,
|
||||
)
|
||||
|
||||
|
||||
class TestChatGPTToolCallNormalizer:
|
||||
"""Test that the normalizer fixes ChatGPT-style tool_call streaming issues."""
|
||||
|
||||
def test_single_tool_call_index_preserved(self):
|
||||
"""A single tool call should get index=0."""
|
||||
chunks = [
|
||||
_make_chunk(
|
||||
tool_calls=[_make_tc(index=0, id="call_1", name="get_weather")]
|
||||
),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='{"loc')]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='ation": "NYC"}')]),
|
||||
]
|
||||
normalizer = ChatGPTToolCallNormalizer(iter(chunks))
|
||||
results = list(normalizer)
|
||||
|
||||
assert len(results) == 3
|
||||
assert results[0].choices[0].delta.tool_calls[0].index == 0
|
||||
assert results[0].choices[0].delta.tool_calls[0].id == "call_1"
|
||||
assert results[1].choices[0].delta.tool_calls[0].index == 0
|
||||
assert results[2].choices[0].delta.tool_calls[0].index == 0
|
||||
|
||||
def test_parallel_tool_calls_get_correct_indices(self):
|
||||
"""
|
||||
ChatGPT sends all tool_calls with index=0. The normalizer should assign
|
||||
sequential indices: 0 for the first, 1 for the second.
|
||||
"""
|
||||
chunks = [
|
||||
# First tool call: intro chunk with id + name
|
||||
_make_chunk(
|
||||
tool_calls=[_make_tc(index=0, id="call_aaa", name="get_weather")]
|
||||
),
|
||||
# First tool call: arguments streaming
|
||||
_make_chunk(
|
||||
tool_calls=[_make_tc(index=0, arguments='{"location": "NYC"}')]
|
||||
),
|
||||
# First tool call: duplicate closing chunk (id repeated) — should be skipped
|
||||
_make_chunk(
|
||||
tool_calls=[_make_tc(index=0, id="call_aaa", name="get_weather")]
|
||||
),
|
||||
# Second tool call: intro chunk with id + name (index=0 from ChatGPT)
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_bbb", name="get_time")]),
|
||||
# Second tool call: arguments streaming
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='{"tz": "EST"}')]),
|
||||
# Second tool call: duplicate closing chunk — should be skipped
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_bbb", name="get_time")]),
|
||||
]
|
||||
|
||||
normalizer = ChatGPTToolCallNormalizer(iter(chunks))
|
||||
results = list(normalizer)
|
||||
|
||||
# 2 duplicate chunks should be skipped → 4 results
|
||||
assert len(results) == 4
|
||||
|
||||
# First tool call chunks should have index=0
|
||||
assert results[0].choices[0].delta.tool_calls[0].index == 0
|
||||
assert results[0].choices[0].delta.tool_calls[0].id == "call_aaa"
|
||||
assert results[1].choices[0].delta.tool_calls[0].index == 0
|
||||
|
||||
# Second tool call chunks should have index=1
|
||||
assert results[2].choices[0].delta.tool_calls[0].index == 1
|
||||
assert results[2].choices[0].delta.tool_calls[0].id == "call_bbb"
|
||||
assert results[3].choices[0].delta.tool_calls[0].index == 1
|
||||
|
||||
def test_non_tool_call_chunks_pass_through(self):
|
||||
"""Chunks without tool_calls should pass through unchanged."""
|
||||
chunks = [
|
||||
_make_chunk(content="Hello"),
|
||||
_make_chunk(content=" world"),
|
||||
]
|
||||
normalizer = ChatGPTToolCallNormalizer(iter(chunks))
|
||||
results = list(normalizer)
|
||||
|
||||
assert len(results) == 2
|
||||
assert results[0].choices[0].delta.content == "Hello"
|
||||
assert results[1].choices[0].delta.content == " world"
|
||||
|
||||
def test_empty_choices_pass_through(self):
|
||||
"""Chunks with empty choices should pass through."""
|
||||
chunk = ModelResponseStream(choices=[])
|
||||
normalizer = ChatGPTToolCallNormalizer(iter([chunk]))
|
||||
results = list(normalizer)
|
||||
|
||||
assert len(results) == 1
|
||||
|
||||
def test_three_parallel_tool_calls(self):
|
||||
"""Three parallel tool calls should get indices 0, 1, 2."""
|
||||
chunks = [
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_1", name="fn_a")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='{"a":1}')]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_2", name="fn_b")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='{"b":2}')]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_3", name="fn_c")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='{"c":3}')]),
|
||||
]
|
||||
|
||||
normalizer = ChatGPTToolCallNormalizer(iter(chunks))
|
||||
results = list(normalizer)
|
||||
|
||||
assert len(results) == 6
|
||||
# First tool call
|
||||
assert results[0].choices[0].delta.tool_calls[0].index == 0
|
||||
assert results[1].choices[0].delta.tool_calls[0].index == 0
|
||||
# Second tool call
|
||||
assert results[2].choices[0].delta.tool_calls[0].index == 1
|
||||
assert results[3].choices[0].delta.tool_calls[0].index == 1
|
||||
# Third tool call
|
||||
assert results[4].choices[0].delta.tool_calls[0].index == 2
|
||||
assert results[5].choices[0].delta.tool_calls[0].index == 2
|
||||
|
||||
def test_all_duplicates_skipped(self):
|
||||
"""If a chunk contains only duplicate tool_calls, the entire chunk is skipped."""
|
||||
chunks = [
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_x", name="fn")]),
|
||||
# Duplicate — same id seen before
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_x", name="fn")]),
|
||||
]
|
||||
|
||||
normalizer = ChatGPTToolCallNormalizer(iter(chunks))
|
||||
results = list(normalizer)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].choices[0].delta.tool_calls[0].id == "call_x"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_iteration(self):
|
||||
"""The normalizer should work with async iteration."""
|
||||
|
||||
async def async_gen():
|
||||
chunks = [
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_a", name="fn_a")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='{"x":1}')]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_b", name="fn_b")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='{"y":2}')]),
|
||||
]
|
||||
for c in chunks:
|
||||
yield c
|
||||
|
||||
normalizer = ChatGPTToolCallNormalizer(async_gen())
|
||||
results = []
|
||||
async for chunk in normalizer:
|
||||
results.append(chunk)
|
||||
|
||||
assert len(results) == 4
|
||||
assert results[0].choices[0].delta.tool_calls[0].index == 0
|
||||
assert results[2].choices[0].delta.tool_calls[0].index == 1
|
||||
|
||||
def test_getattr_proxies_to_stream(self):
|
||||
"""Attribute access should be proxied to the underlying stream."""
|
||||
|
||||
class FakeStream:
|
||||
custom_attr = "test_value"
|
||||
|
||||
def __iter__(self):
|
||||
return iter([])
|
||||
|
||||
def __next__(self):
|
||||
raise StopIteration
|
||||
|
||||
normalizer = ChatGPTToolCallNormalizer(FakeStream())
|
||||
assert normalizer.custom_attr == "test_value"
|
||||
|
||||
|
||||
def test_chatgpt_config_streams_parallel_tool_calls_with_distinct_indices(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path))
|
||||
chunks: Final = (
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_aaa", name="get_weather")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_bbb", name="get_time")]),
|
||||
)
|
||||
|
||||
results: Final = list(ChatGPTConfig().post_stream_processing(iter(chunks)))
|
||||
|
||||
assert [result.choices[0].delta.tool_calls[0].index for result in results] == [0, 1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source_indices", [(2, 7), (0, 1)])
|
||||
def test_chatgpt_consumer_reassembles_interleaved_tool_call_arguments_by_source_index(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, source_indices: tuple[int, int]
|
||||
):
|
||||
monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path))
|
||||
weather_index: Final = source_indices[0]
|
||||
time_index: Final = source_indices[1]
|
||||
chunks: Final = (
|
||||
_make_chunk(tool_calls=[_make_tc(index=weather_index, id="call_weather", name="get_weather")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=time_index, id="call_time", name="get_time")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=time_index, arguments='{"timezone":')]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=weather_index, arguments='{"location":')]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=time_index, arguments='"EST"}')]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=weather_index, arguments='"NYC"}')]),
|
||||
)
|
||||
|
||||
results: Final = list(ChatGPTConfig().post_stream_processing(iter(chunks)))
|
||||
tool_calls: Final = tuple(result.choices[0].delta.tool_calls[0] for result in results)
|
||||
|
||||
assert {
|
||||
tool_call.function.name: json.loads(
|
||||
"".join(delta.function.arguments or "" for delta in tool_calls if delta.index == tool_call.index)
|
||||
)
|
||||
for tool_call in tool_calls
|
||||
if tool_call.id
|
||||
} == {"get_weather": {"location": "NYC"}, "get_time": {"timezone": "EST"}}
|
||||
"""
|
||||
Tests for ChatGPTToolCallNormalizer.
|
||||
|
||||
Verifies that non-spec-compliant tool_call chunks from the ChatGPT backend API
|
||||
are normalized to match the OpenAI streaming spec:
|
||||
- Correct index assignment for parallel tool calls
|
||||
- Deduplication of "closing" chunks with repeated id/name
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.chatgpt.chat.streaming_utils import ChatGPTToolCallNormalizer
|
||||
from litellm.llms.chatgpt.chat.transformation import ChatGPTConfig
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionDeltaToolCall,
|
||||
Delta,
|
||||
Function,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
)
|
||||
|
||||
|
||||
def _make_chunk(tool_calls=None, content=None):
|
||||
"""Helper to build a ModelResponseStream chunk with tool_calls on the delta."""
|
||||
delta = Delta(
|
||||
content=content,
|
||||
role="assistant",
|
||||
tool_calls=tool_calls,
|
||||
)
|
||||
choice = StreamingChoices(delta=delta, index=0)
|
||||
return ModelResponseStream(choices=[choice])
|
||||
|
||||
|
||||
def _make_tc(index=0, id=None, name=None, arguments=None):
|
||||
"""Helper to build a ChatCompletionDeltaToolCall."""
|
||||
func = Function(name=name, arguments=arguments)
|
||||
return ChatCompletionDeltaToolCall(
|
||||
index=index,
|
||||
id=id,
|
||||
function=func,
|
||||
type="function" if id else None,
|
||||
)
|
||||
|
||||
|
||||
class TestChatGPTToolCallNormalizer:
|
||||
"""Test that the normalizer fixes ChatGPT-style tool_call streaming issues."""
|
||||
|
||||
def test_single_tool_call_index_preserved(self):
|
||||
"""A single tool call should get index=0."""
|
||||
chunks = [
|
||||
_make_chunk(
|
||||
tool_calls=[_make_tc(index=0, id="call_1", name="get_weather")]
|
||||
),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='{"loc')]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='ation": "NYC"}')]),
|
||||
]
|
||||
normalizer = ChatGPTToolCallNormalizer(iter(chunks))
|
||||
results = list(normalizer)
|
||||
|
||||
assert len(results) == 3
|
||||
assert results[0].choices[0].delta.tool_calls[0].index == 0
|
||||
assert results[0].choices[0].delta.tool_calls[0].id == "call_1"
|
||||
assert results[1].choices[0].delta.tool_calls[0].index == 0
|
||||
assert results[2].choices[0].delta.tool_calls[0].index == 0
|
||||
|
||||
def test_parallel_tool_calls_get_correct_indices(self):
|
||||
"""
|
||||
ChatGPT sends all tool_calls with index=0. The normalizer should assign
|
||||
sequential indices: 0 for the first, 1 for the second.
|
||||
"""
|
||||
chunks = [
|
||||
# First tool call: intro chunk with id + name
|
||||
_make_chunk(
|
||||
tool_calls=[_make_tc(index=0, id="call_aaa", name="get_weather")]
|
||||
),
|
||||
# First tool call: arguments streaming
|
||||
_make_chunk(
|
||||
tool_calls=[_make_tc(index=0, arguments='{"location": "NYC"}')]
|
||||
),
|
||||
# First tool call: duplicate closing chunk (id repeated) — should be skipped
|
||||
_make_chunk(
|
||||
tool_calls=[_make_tc(index=0, id="call_aaa", name="get_weather")]
|
||||
),
|
||||
# Second tool call: intro chunk with id + name (index=0 from ChatGPT)
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_bbb", name="get_time")]),
|
||||
# Second tool call: arguments streaming
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='{"tz": "EST"}')]),
|
||||
# Second tool call: duplicate closing chunk — should be skipped
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_bbb", name="get_time")]),
|
||||
]
|
||||
|
||||
normalizer = ChatGPTToolCallNormalizer(iter(chunks))
|
||||
results = list(normalizer)
|
||||
|
||||
# 2 duplicate chunks should be skipped → 4 results
|
||||
assert len(results) == 4
|
||||
|
||||
# First tool call chunks should have index=0
|
||||
assert results[0].choices[0].delta.tool_calls[0].index == 0
|
||||
assert results[0].choices[0].delta.tool_calls[0].id == "call_aaa"
|
||||
assert results[1].choices[0].delta.tool_calls[0].index == 0
|
||||
|
||||
# Second tool call chunks should have index=1
|
||||
assert results[2].choices[0].delta.tool_calls[0].index == 1
|
||||
assert results[2].choices[0].delta.tool_calls[0].id == "call_bbb"
|
||||
assert results[3].choices[0].delta.tool_calls[0].index == 1
|
||||
|
||||
def test_non_tool_call_chunks_pass_through(self):
|
||||
"""Chunks without tool_calls should pass through unchanged."""
|
||||
chunks = [
|
||||
_make_chunk(content="Hello"),
|
||||
_make_chunk(content=" world"),
|
||||
]
|
||||
normalizer = ChatGPTToolCallNormalizer(iter(chunks))
|
||||
results = list(normalizer)
|
||||
|
||||
assert len(results) == 2
|
||||
assert results[0].choices[0].delta.content == "Hello"
|
||||
assert results[1].choices[0].delta.content == " world"
|
||||
|
||||
def test_empty_choices_pass_through(self):
|
||||
"""Chunks with empty choices should pass through."""
|
||||
chunk = ModelResponseStream(choices=[])
|
||||
normalizer = ChatGPTToolCallNormalizer(iter([chunk]))
|
||||
results = list(normalizer)
|
||||
|
||||
assert len(results) == 1
|
||||
|
||||
def test_three_parallel_tool_calls(self):
|
||||
"""Three parallel tool calls should get indices 0, 1, 2."""
|
||||
chunks = [
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_1", name="fn_a")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='{"a":1}')]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_2", name="fn_b")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='{"b":2}')]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_3", name="fn_c")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='{"c":3}')]),
|
||||
]
|
||||
|
||||
normalizer = ChatGPTToolCallNormalizer(iter(chunks))
|
||||
results = list(normalizer)
|
||||
|
||||
assert len(results) == 6
|
||||
# First tool call
|
||||
assert results[0].choices[0].delta.tool_calls[0].index == 0
|
||||
assert results[1].choices[0].delta.tool_calls[0].index == 0
|
||||
# Second tool call
|
||||
assert results[2].choices[0].delta.tool_calls[0].index == 1
|
||||
assert results[3].choices[0].delta.tool_calls[0].index == 1
|
||||
# Third tool call
|
||||
assert results[4].choices[0].delta.tool_calls[0].index == 2
|
||||
assert results[5].choices[0].delta.tool_calls[0].index == 2
|
||||
|
||||
def test_all_duplicates_skipped(self):
|
||||
"""If a chunk contains only duplicate tool_calls, the entire chunk is skipped."""
|
||||
chunks = [
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_x", name="fn")]),
|
||||
# Duplicate — same id seen before
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_x", name="fn")]),
|
||||
]
|
||||
|
||||
normalizer = ChatGPTToolCallNormalizer(iter(chunks))
|
||||
results = list(normalizer)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].choices[0].delta.tool_calls[0].id == "call_x"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_iteration(self):
|
||||
"""The normalizer should work with async iteration."""
|
||||
|
||||
async def async_gen():
|
||||
chunks = [
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_a", name="fn_a")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='{"x":1}')]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_b", name="fn_b")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, arguments='{"y":2}')]),
|
||||
]
|
||||
for c in chunks:
|
||||
yield c
|
||||
|
||||
normalizer = ChatGPTToolCallNormalizer(async_gen())
|
||||
results = []
|
||||
async for chunk in normalizer:
|
||||
results.append(chunk)
|
||||
|
||||
assert len(results) == 4
|
||||
assert results[0].choices[0].delta.tool_calls[0].index == 0
|
||||
assert results[2].choices[0].delta.tool_calls[0].index == 1
|
||||
|
||||
def test_getattr_proxies_to_stream(self):
|
||||
"""Attribute access should be proxied to the underlying stream."""
|
||||
|
||||
class FakeStream:
|
||||
custom_attr = "test_value"
|
||||
|
||||
def __iter__(self):
|
||||
return iter([])
|
||||
|
||||
def __next__(self):
|
||||
raise StopIteration
|
||||
|
||||
normalizer = ChatGPTToolCallNormalizer(FakeStream())
|
||||
assert normalizer.custom_attr == "test_value"
|
||||
|
||||
|
||||
def test_chatgpt_config_streams_parallel_tool_calls_with_distinct_indices(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path))
|
||||
chunks: Final = (
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_aaa", name="get_weather")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=0, id="call_bbb", name="get_time")]),
|
||||
)
|
||||
|
||||
results: Final = list(ChatGPTConfig().post_stream_processing(iter(chunks)))
|
||||
|
||||
assert [result.choices[0].delta.tool_calls[0].index for result in results] == [0, 1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source_indices", [(2, 7), (0, 1)])
|
||||
def test_chatgpt_consumer_reassembles_interleaved_tool_call_arguments_by_source_index(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, source_indices: tuple[int, int]
|
||||
):
|
||||
monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path))
|
||||
weather_index: Final = source_indices[0]
|
||||
time_index: Final = source_indices[1]
|
||||
chunks: Final = (
|
||||
_make_chunk(tool_calls=[_make_tc(index=weather_index, id="call_weather", name="get_weather")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=time_index, id="call_time", name="get_time")]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=time_index, arguments='{"timezone":')]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=weather_index, arguments='{"location":')]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=time_index, arguments='"EST"}')]),
|
||||
_make_chunk(tool_calls=[_make_tc(index=weather_index, arguments='"NYC"}')]),
|
||||
)
|
||||
|
||||
results: Final = list(ChatGPTConfig().post_stream_processing(iter(chunks)))
|
||||
tool_calls: Final = tuple(result.choices[0].delta.tool_calls[0] for result in results)
|
||||
|
||||
assert {
|
||||
tool_call.function.name: json.loads(
|
||||
"".join(delta.function.arguments or "" for delta in tool_calls if delta.index == tool_call.index)
|
||||
)
|
||||
for tool_call in tool_calls
|
||||
if tool_call.id
|
||||
} == {"get_weather": {"location": "NYC"}, "get_time": {"timezone": "EST"}}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue