mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-05 07:37:50 +00:00
Merge c6e4d372cd into f1795e6d44
This commit is contained in:
raja
GitHub
commit
41c4aa0342
5 changed files with 677 additions and 35 deletions
|
|
@ -971,6 +971,11 @@ class CustomStreamWrapper:
|
|||
if (
|
||||
"content" in completion_obj
|
||||
and (isinstance(completion_obj["content"], str) and len(completion_obj["content"]) > 0)
|
||||
or (
|
||||
"reasoning_content" in completion_obj
|
||||
and completion_obj["reasoning_content"] is not None
|
||||
and len(completion_obj["reasoning_content"]) > 0
|
||||
)
|
||||
or (
|
||||
"tool_calls" in completion_obj
|
||||
and completion_obj["tool_calls"] is not None
|
||||
|
|
@ -1214,7 +1219,12 @@ class CustomStreamWrapper:
|
|||
)
|
||||
else:
|
||||
## else
|
||||
completion_obj["content"] = model_response_str
|
||||
if model_response_str or (
|
||||
completion_obj.get("reasoning_content") is None and completion_obj.get("tool_calls") is None
|
||||
):
|
||||
completion_obj["content"] = model_response_str # rebind-ok: builds delta payload
|
||||
else:
|
||||
completion_obj["content"] = None # rebind-ok: builds delta payload
|
||||
if self.sent_first_chunk is False:
|
||||
completion_obj["role"] = "assistant"
|
||||
self.sent_first_chunk = True
|
||||
|
|
@ -1345,7 +1355,9 @@ class CustomStreamWrapper:
|
|||
if self.received_finish_reason is not None:
|
||||
_chunk_has_content: Final = isinstance(chunk, dict) and (
|
||||
bool(chunk.get("text", ""))
|
||||
or bool(chunk.get("reasoning_content", ""))
|
||||
or chunk.get("tool_use") is not None
|
||||
or bool(chunk.get("tool_calls"))
|
||||
# Usage-only final chunks are valid and needed to surface
|
||||
# finish_reason/usage to downstream translators.
|
||||
or chunk.get("usage") is not None
|
||||
|
|
@ -1354,6 +1366,9 @@ class CustomStreamWrapper:
|
|||
raise StopIteration
|
||||
anthropic_response_obj: Final[GChunk] = cast(GChunk, chunk)
|
||||
completion_obj["content"] = anthropic_response_obj["text"]
|
||||
reasoning_chunk: Final = anthropic_response_obj.get("reasoning_content")
|
||||
if reasoning_chunk is not None:
|
||||
completion_obj["reasoning_content"] = reasoning_chunk # rebind-ok: forwards reasoning
|
||||
if anthropic_response_obj["is_finished"]:
|
||||
self.received_finish_reason = anthropic_response_obj["finish_reason"]
|
||||
|
||||
|
|
@ -1367,8 +1382,12 @@ class CustomStreamWrapper:
|
|||
litellm.Usage(**anthropic_response_obj["usage"]),
|
||||
)
|
||||
|
||||
if "tool_use" in anthropic_response_obj and anthropic_response_obj["tool_use"] is not None:
|
||||
completion_obj["tool_calls"] = [anthropic_response_obj["tool_use"]]
|
||||
tool_calls_chunk: Final = anthropic_response_obj.get("tool_calls")
|
||||
legacy_tool_chunk: Final = anthropic_response_obj.get("tool_use")
|
||||
if tool_calls_chunk is not None:
|
||||
completion_obj["tool_calls"] = tool_calls_chunk # rebind-ok: forwards tool calls
|
||||
elif legacy_tool_chunk is not None:
|
||||
completion_obj["tool_calls"] = [legacy_tool_chunk] # rebind-ok: forwards tool calls
|
||||
|
||||
if (
|
||||
"provider_specific_fields" in anthropic_response_obj
|
||||
|
|
@ -2781,6 +2800,17 @@ def convert_generic_chunk_to_model_response_stream(
|
|||
) -> ModelResponseStream:
|
||||
from litellm.types.utils import Delta
|
||||
|
||||
raw_tool_calls: Final = chunk.get("tool_calls", None)
|
||||
legacy_tool_use: Final = chunk.get("tool_use", None)
|
||||
tool_calls: Final = [legacy_tool_use] if raw_tool_calls is None and legacy_tool_use is not None else raw_tool_calls
|
||||
|
||||
raw_content: Final = chunk["text"]
|
||||
content: Final = (
|
||||
None
|
||||
if not raw_content and (chunk.get("reasoning_content") is not None or tool_calls is not None)
|
||||
else raw_content
|
||||
)
|
||||
|
||||
model_response_stream: Final = ModelResponseStream(
|
||||
id=str(uuid.uuid4()),
|
||||
model="",
|
||||
|
|
@ -2788,8 +2818,9 @@ def convert_generic_chunk_to_model_response_stream(
|
|||
StreamingChoices(
|
||||
index=chunk.get("index", 0),
|
||||
delta=Delta(
|
||||
content=chunk["text"],
|
||||
tool_calls=chunk.get("tool_use", None),
|
||||
content=content,
|
||||
reasoning_content=chunk.get("reasoning_content", None),
|
||||
tool_calls=tool_calls,
|
||||
),
|
||||
)
|
||||
],
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
|
|
@ -31,12 +32,6 @@ class ModelResponseIterator:
|
|||
try:
|
||||
processed_chunk: Final = litellm.ModelResponseStream(**chunk)
|
||||
|
||||
text = ""
|
||||
tool_use: ChatCompletionToolCallChunk | None = None
|
||||
is_finished = False
|
||||
finish_reason = ""
|
||||
usage: ChatCompletionUsageBlock | None = None
|
||||
|
||||
# Usage-only final chunk (OpenAI ``stream_options.include_usage``)
|
||||
# arrives with an empty ``choices`` list — return usage without
|
||||
# indexing ``choices[0]``.
|
||||
|
|
@ -51,36 +46,47 @@ class ModelResponseIterator:
|
|||
index=0,
|
||||
)
|
||||
|
||||
if processed_chunk.choices[0].delta.content is not None:
|
||||
text = processed_chunk.choices[0].delta.content
|
||||
choice_0: Final = processed_chunk.choices[0]
|
||||
choice_delta: Final = choice_0.delta
|
||||
text: Final[str] = choice_delta.content if choice_delta.content is not None else ""
|
||||
reasoning_content: Final[str | None] = getattr(choice_delta, "reasoning_content", None)
|
||||
|
||||
if (
|
||||
processed_chunk.choices[0].delta.tool_calls is not None
|
||||
and len(processed_chunk.choices[0].delta.tool_calls) > 0
|
||||
and processed_chunk.choices[0].delta.tool_calls[0].function is not None
|
||||
and processed_chunk.choices[0].delta.tool_calls[0].function.arguments is not None
|
||||
):
|
||||
tool_use = ChatCompletionToolCallChunk(
|
||||
id=processed_chunk.choices[0].delta.tool_calls[0].id,
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=processed_chunk.choices[0].delta.tool_calls[0].function.name,
|
||||
arguments=processed_chunk.choices[0].delta.tool_calls[0].function.arguments,
|
||||
),
|
||||
index=processed_chunk.choices[0].delta.tool_calls[0].index,
|
||||
)
|
||||
tool_calls_list: list[ChatCompletionToolCallChunk] = [] # mutable-ok: accumulates delta calls
|
||||
if choice_delta.tool_calls is not None:
|
||||
for tc in choice_delta.tool_calls:
|
||||
tc_fn = getattr(tc, "function", None)
|
||||
fn_chunk = (
|
||||
ChatCompletionToolCallFunctionChunk(
|
||||
name=getattr(tc_fn, "name", None),
|
||||
arguments=getattr(tc_fn, "arguments", "") or "",
|
||||
)
|
||||
if tc_fn is not None
|
||||
else ChatCompletionToolCallFunctionChunk(name=None, arguments="")
|
||||
)
|
||||
call_chunk = ChatCompletionToolCallChunk(
|
||||
id=getattr(tc, "id", None),
|
||||
type="function",
|
||||
function=fn_chunk,
|
||||
index=getattr(tc, "index", 0) if getattr(tc, "index", None) is not None else 0,
|
||||
)
|
||||
tool_calls_list.append(call_chunk)
|
||||
|
||||
if processed_chunk.choices[0].finish_reason is not None:
|
||||
is_finished = True
|
||||
finish_reason = processed_chunk.choices[0].finish_reason
|
||||
tool_calls: Final[Sequence[ChatCompletionToolCallChunk] | None] = tool_calls_list or None
|
||||
tool_use: Final[ChatCompletionToolCallChunk | None] = tool_calls_list[0] if tool_calls_list else None
|
||||
|
||||
is_finished: Final[bool] = choice_0.finish_reason is not None
|
||||
finish_reason: Final[str] = choice_0.finish_reason or ""
|
||||
|
||||
usage_chunk: Final[Usage | None] = getattr(processed_chunk, "usage", None)
|
||||
if usage_chunk is not None:
|
||||
usage = _usage_block(usage_chunk)
|
||||
usage: Final[ChatCompletionUsageBlock | None] = (
|
||||
_usage_block(usage_chunk) if usage_chunk is not None else None
|
||||
)
|
||||
|
||||
return GenericStreamingChunk(
|
||||
text=text,
|
||||
tool_use=tool_use,
|
||||
tool_calls=tool_calls,
|
||||
reasoning_content=reasoning_content,
|
||||
is_finished=is_finished,
|
||||
finish_reason=finish_reason,
|
||||
usage=usage,
|
||||
|
|
|
|||
|
|
@ -498,6 +498,8 @@ class ModelInfo(ModelInfoBase, total=False):
|
|||
class GenericStreamingChunk(TypedDict, total=False):
|
||||
text: Required[str]
|
||||
tool_use: ChatCompletionToolCallChunk | None
|
||||
tool_calls: ReadOnly[Sequence[ChatCompletionToolCallChunk] | None]
|
||||
reasoning_content: ReadOnly[str | None]
|
||||
is_finished: Required[bool]
|
||||
finish_reason: Required[str]
|
||||
usage: Required[ChatCompletionUsageBlock | None]
|
||||
|
|
|
|||
|
|
@ -182,7 +182,9 @@ async def test_async_stream_skips_blank_sse_lines_and_keeps_the_usage_chunk():
|
|||
|
||||
chunks: Final = [chunk async for chunk in ModelResponseIterator(streaming_response=lines(), sync_stream=False)]
|
||||
|
||||
assert [(chunk["text"], chunk["finish_reason"], (chunk["usage"] or {}).get("prompt_tokens")) for chunk in chunks] == [
|
||||
assert [
|
||||
(chunk["text"], chunk["finish_reason"], (chunk["usage"] or {}).get("prompt_tokens")) for chunk in chunks
|
||||
] == [
|
||||
("144", "", None),
|
||||
("", "stop", None),
|
||||
("", "", None),
|
||||
|
|
@ -222,3 +224,72 @@ async def test_async_stream_without_done_marker_ends_after_the_last_chunk():
|
|||
chunks: Final = [chunk async for chunk in ModelResponseIterator(streaming_response=lines(), sync_stream=False)]
|
||||
|
||||
assert [(chunk["text"], chunk["finish_reason"]) for chunk in chunks] == [("144", ""), ("", "stop")]
|
||||
|
||||
|
||||
def test_chunk_parser_extracts_reasoning_content():
|
||||
"""Reasoning-only delta chunks must preserve reasoning_content."""
|
||||
iterator = ModelResponseIterator(streaming_response=None, sync_stream=True)
|
||||
chunk = {
|
||||
"id": "chatcmpl-reasoning",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"role": "assistant", "reasoning_content": "Thinking about prime numbers..."},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
result = iterator.chunk_parser(chunk=chunk)
|
||||
|
||||
assert result["text"] == ""
|
||||
assert result.get("reasoning_content") == "Thinking about prime numbers..."
|
||||
|
||||
|
||||
def test_chunk_parser_extracts_multiple_tool_calls_with_indices():
|
||||
"""Multiple tool call deltas in a chunk must preserve all tool calls and their indices."""
|
||||
iterator = ModelResponseIterator(streaming_response=None, sync_stream=True)
|
||||
chunk = {
|
||||
"id": "chatcmpl-tools",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'},
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"id": "call_2",
|
||||
"type": "function",
|
||||
"function": {"name": "get_time", "arguments": '{"city": "Paris"}'},
|
||||
},
|
||||
]
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
result = iterator.chunk_parser(chunk=chunk)
|
||||
|
||||
assert result.get("tool_calls") is not None
|
||||
assert len(result["tool_calls"]) == 2
|
||||
assert result["tool_calls"][0]["id"] == "call_1"
|
||||
assert result["tool_calls"][0]["function"]["name"] == "get_weather"
|
||||
assert result["tool_calls"][0]["index"] == 0
|
||||
assert result["tool_calls"][1]["id"] == "call_2"
|
||||
assert result["tool_calls"][1]["function"]["name"] == "get_time"
|
||||
assert result["tool_calls"][1]["index"] == 1
|
||||
# Backward compatibility with tool_use
|
||||
assert result["tool_use"]["id"] == "call_1"
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.llms.openai_like.chat.handler import make_sync_call
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
|
||||
_FAKE_STREAMED_REPLY: Final = {
|
||||
"id": "chatcmpl-fake-stream",
|
||||
|
|
@ -73,3 +73,535 @@ def test_fake_stream_keeps_reasoning_every_tool_call_and_token_detail():
|
|||
assert [call.function.name for call in delta.tool_calls] == ["get_weather", "get_time"]
|
||||
assert chunks[0].usage.prompt_tokens_details.cached_tokens == 64
|
||||
assert chunks[0].usage.completion_tokens_details.reasoning_tokens == 1920
|
||||
|
||||
|
||||
def test_real_stream_keeps_reasoning_multi_tool_calls_and_terminal_usage():
|
||||
events = [
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "c1",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"role": "assistant", "reasoning_content": "Thinking step 1"},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "c2",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'},
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"id": "call_2",
|
||||
"type": "function",
|
||||
"function": {"name": "get_time", "arguments": '{"city": "Paris"}'},
|
||||
},
|
||||
]
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "c3",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"content": "Here is the result"},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "c4",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
|
||||
}
|
||||
),
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "c5",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 64,
|
||||
"total_tokens": 74,
|
||||
"completion_tokens_details": {"reasoning_tokens": 30},
|
||||
},
|
||||
}
|
||||
),
|
||||
"data: [DONE]",
|
||||
]
|
||||
sse_bytes = ("\n\n".join(events) + "\n\n").encode("utf-8")
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=sse_bytes)
|
||||
|
||||
mock_logging = Mock()
|
||||
mock_logging.model_call_details = {"litellm_params": {}}
|
||||
mock_logging.stream_options = {"include_usage": True}
|
||||
|
||||
raw_stream = make_sync_call(
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))),
|
||||
api_base="https://openai-like.example/v1/chat/completions",
|
||||
headers={},
|
||||
data=json.dumps({"model": "glm-4", "messages": []}),
|
||||
model="glm-4",
|
||||
messages=[],
|
||||
logging_obj=mock_logging,
|
||||
fake_stream=False,
|
||||
)
|
||||
wrapper = CustomStreamWrapper(
|
||||
completion_stream=raw_stream,
|
||||
model="glm-4",
|
||||
custom_llm_provider="vertex_ai",
|
||||
logging_obj=mock_logging,
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
|
||||
chunks = list(wrapper)
|
||||
assert len(chunks) == 5
|
||||
|
||||
# 1. Reasoning-only delta must reach the client with reasoning_content and content=None
|
||||
reasoning_chunk = chunks[0]
|
||||
assert reasoning_chunk.choices[0].delta.reasoning_content == "Thinking step 1"
|
||||
assert reasoning_chunk.choices[0].delta.content is None
|
||||
|
||||
# 2. Multi tool call delta must preserve both tool calls and their indices
|
||||
tool_chunk = chunks[1]
|
||||
assert len(tool_chunk.choices[0].delta.tool_calls) == 2
|
||||
assert tool_chunk.choices[0].delta.tool_calls[0].function.name == "get_weather"
|
||||
assert tool_chunk.choices[0].delta.tool_calls[0].index == 0
|
||||
assert tool_chunk.choices[0].delta.tool_calls[1].function.name == "get_time"
|
||||
assert tool_chunk.choices[0].delta.tool_calls[1].index == 1
|
||||
|
||||
# 3. Content delta
|
||||
assert chunks[2].choices[0].delta.content == "Here is the result"
|
||||
|
||||
# 4. Finish reason
|
||||
assert chunks[3].choices[0].finish_reason == "stop"
|
||||
|
||||
# 5. Terminal usage chunk with provider usage details preserved
|
||||
assert chunks[4].usage is not None
|
||||
assert chunks[4].usage.completion_tokens == 64
|
||||
assert chunks[4].usage.prompt_tokens == 10
|
||||
assert chunks[4].usage.completion_tokens_details.reasoning_tokens == 30
|
||||
|
||||
|
||||
def test_real_stream_incremental_alternating_tool_calls_and_negative_controls():
|
||||
events: Final[list[str]] = [
|
||||
# Frame 1: index 0 initial (id, name, arg prefix)
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "c1",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call_weather",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": '{"city": "'},
|
||||
}
|
||||
]
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
# Frame 2: index 1 initial (id, name, arg prefix)
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "c2",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 1,
|
||||
"id": "call_time",
|
||||
"type": "function",
|
||||
"function": {"name": "get_time", "arguments": '{"location": "'},
|
||||
}
|
||||
]
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
# Frame 3: index 0 suffix (id and name omitted)
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "c3",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"function": {"arguments": 'Paris"}'},
|
||||
}
|
||||
]
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
# Frame 4: index 1 suffix (id and name omitted)
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "c4",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 1,
|
||||
"function": {"arguments": 'Europe/Paris"}'},
|
||||
}
|
||||
]
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
# Frame 5: normal finish_reason length as reporter observed on Vertex
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "c5",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "length"}],
|
||||
}
|
||||
),
|
||||
# Frame 6: empty-choice usage-only terminal completion_tokens=64
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "c6",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 12,
|
||||
"completion_tokens": 64,
|
||||
"total_tokens": 76,
|
||||
"completion_tokens_details": {"reasoning_tokens": 18},
|
||||
},
|
||||
}
|
||||
),
|
||||
"data: [DONE]",
|
||||
]
|
||||
|
||||
sse_bytes: Final[bytes] = ("\n\n".join(events) + "\n\n").encode("utf-8")
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=sse_bytes)
|
||||
|
||||
mock_logging: Final = Mock()
|
||||
mock_logging.model_call_details = {"litellm_params": {}}
|
||||
mock_logging.stream_options = {"include_usage": True}
|
||||
|
||||
raw_stream: Final = make_sync_call(
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))),
|
||||
api_base="https://openai-like.example/v1/chat/completions",
|
||||
headers={},
|
||||
data=json.dumps({"model": "glm-4", "messages": []}),
|
||||
model="glm-4",
|
||||
messages=[],
|
||||
logging_obj=mock_logging,
|
||||
fake_stream=False,
|
||||
)
|
||||
wrapper: Final = CustomStreamWrapper(
|
||||
completion_stream=raw_stream,
|
||||
model="glm-4",
|
||||
custom_llm_provider="vertex_ai",
|
||||
logging_obj=mock_logging,
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
|
||||
chunks: Final = list(wrapper)
|
||||
assert len(chunks) == 6
|
||||
|
||||
# Assert per-frame index/id/argument preservation
|
||||
# Frame 0: index 0 initial
|
||||
assert chunks[0].choices[0].delta.tool_calls[0].index == 0
|
||||
assert chunks[0].choices[0].delta.tool_calls[0].id == "call_weather"
|
||||
assert chunks[0].choices[0].delta.tool_calls[0].function.name == "get_weather"
|
||||
assert chunks[0].choices[0].delta.tool_calls[0].function.arguments == '{"city": "'
|
||||
|
||||
# Frame 1: index 1 initial
|
||||
assert chunks[1].choices[0].delta.tool_calls[0].index == 1
|
||||
assert chunks[1].choices[0].delta.tool_calls[0].id == "call_time"
|
||||
assert chunks[1].choices[0].delta.tool_calls[0].function.name == "get_time"
|
||||
assert chunks[1].choices[0].delta.tool_calls[0].function.arguments == '{"location": "'
|
||||
|
||||
# Frame 2: index 0 continuation (id/name absent)
|
||||
assert chunks[2].choices[0].delta.tool_calls[0].index == 0
|
||||
assert chunks[2].choices[0].delta.tool_calls[0].id is None
|
||||
assert chunks[2].choices[0].delta.tool_calls[0].function.name is None
|
||||
assert chunks[2].choices[0].delta.tool_calls[0].function.arguments == 'Paris"}'
|
||||
|
||||
# Frame 3: index 1 continuation (id/name absent)
|
||||
assert chunks[3].choices[0].delta.tool_calls[0].index == 1
|
||||
assert chunks[3].choices[0].delta.tool_calls[0].id is None
|
||||
assert chunks[3].choices[0].delta.tool_calls[0].function.name is None
|
||||
assert chunks[3].choices[0].delta.tool_calls[0].function.arguments == 'Europe/Paris"}'
|
||||
|
||||
# Frame 4: finish_reason length
|
||||
assert chunks[4].choices[0].finish_reason == "length"
|
||||
|
||||
# Frame 5: usage chunk
|
||||
assert chunks[5].usage is not None
|
||||
assert chunks[5].usage.completion_tokens == 64
|
||||
assert chunks[5].usage.prompt_tokens == 12
|
||||
|
||||
# Reassembling in client: final separately ordered complete arguments
|
||||
assembled_calls: Final[dict[int, dict[str, str]]] = {}
|
||||
for c in chunks:
|
||||
if c.choices and c.choices[0].delta and c.choices[0].delta.tool_calls:
|
||||
for tc in c.choices[0].delta.tool_calls:
|
||||
idx = tc.index
|
||||
if idx not in assembled_calls:
|
||||
assembled_calls[idx] = {"id": tc.id or "", "name": tc.function.name or "", "arguments": ""}
|
||||
if tc.id:
|
||||
assembled_calls[idx]["id"] = tc.id
|
||||
if tc.function.name:
|
||||
assembled_calls[idx]["name"] = tc.function.name
|
||||
assembled_calls[idx]["arguments"] += tc.function.arguments
|
||||
|
||||
assert assembled_calls[0]["id"] == "call_weather"
|
||||
assert assembled_calls[0]["name"] == "get_weather"
|
||||
assert assembled_calls[0]["arguments"] == '{"city": "Paris"}'
|
||||
|
||||
assert assembled_calls[1]["id"] == "call_time"
|
||||
assert assembled_calls[1]["name"] == "get_time"
|
||||
assert assembled_calls[1]["arguments"] == '{"location": "Europe/Paris"}'
|
||||
|
||||
|
||||
def test_real_stream_content_only_and_no_provider_usage_negative_controls():
|
||||
# Negative control 1: content-only stream emits no ghost tool calls, reasoning, or provider usage
|
||||
events_content: Final[list[str]] = [
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "c1",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"content": "Direct text reply."},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "c2",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
|
||||
}
|
||||
),
|
||||
"data: [DONE]",
|
||||
]
|
||||
sse_bytes_content: Final[bytes] = ("\n\n".join(events_content) + "\n\n").encode("utf-8")
|
||||
|
||||
def respond_content(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=sse_bytes_content)
|
||||
|
||||
mock_logging_content: Final = Mock()
|
||||
mock_logging_content.model_call_details = {"litellm_params": {}}
|
||||
mock_logging_content.stream_options = {}
|
||||
|
||||
stream_content: Final = make_sync_call(
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond_content))),
|
||||
api_base="https://openai-like.example/v1/chat/completions",
|
||||
headers={},
|
||||
data=json.dumps({"model": "glm-4", "messages": []}),
|
||||
model="glm-4",
|
||||
messages=[],
|
||||
logging_obj=mock_logging_content,
|
||||
fake_stream=False,
|
||||
)
|
||||
wrapper_content: Final = CustomStreamWrapper(
|
||||
completion_stream=stream_content,
|
||||
model="glm-4",
|
||||
custom_llm_provider="vertex_ai",
|
||||
logging_obj=mock_logging_content,
|
||||
stream_options={},
|
||||
)
|
||||
chunks_content: Final = list(wrapper_content)
|
||||
assert len(chunks_content) == 2
|
||||
assert chunks_content[0].choices[0].delta.content == "Direct text reply."
|
||||
assert chunks_content[0].choices[0].delta.tool_calls is None
|
||||
assert getattr(chunks_content[0].choices[0].delta, "reasoning_content", None) is None
|
||||
assert getattr(chunks_content[0], "usage", None) is None
|
||||
assert chunks_content[1].choices[0].finish_reason == "stop"
|
||||
|
||||
# Negative control 2: no-provider-usage stream terminates cleanly without synthetic usage chunks
|
||||
events_no_usage: Final[list[str]] = [
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "c1",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call_single",
|
||||
"type": "function",
|
||||
"function": {"name": "lookup", "arguments": "{}"},
|
||||
}
|
||||
]
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "c2",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-4",
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}],
|
||||
}
|
||||
),
|
||||
"data: [DONE]",
|
||||
]
|
||||
sse_bytes_no_usage: Final[bytes] = ("\n\n".join(events_no_usage) + "\n\n").encode("utf-8")
|
||||
|
||||
def respond_no_usage(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=sse_bytes_no_usage)
|
||||
|
||||
mock_logging_no_usage: Final = Mock()
|
||||
mock_logging_no_usage.model_call_details = {"litellm_params": {}}
|
||||
mock_logging_no_usage.stream_options = {}
|
||||
|
||||
stream_no_usage: Final = make_sync_call(
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond_no_usage))),
|
||||
api_base="https://openai-like.example/v1/chat/completions",
|
||||
headers={},
|
||||
data=json.dumps({"model": "glm-4", "messages": []}),
|
||||
model="glm-4",
|
||||
messages=[],
|
||||
logging_obj=mock_logging_no_usage,
|
||||
fake_stream=False,
|
||||
)
|
||||
wrapper_no_usage: Final = CustomStreamWrapper(
|
||||
completion_stream=stream_no_usage,
|
||||
model="glm-4",
|
||||
custom_llm_provider="vertex_ai",
|
||||
logging_obj=mock_logging_no_usage,
|
||||
stream_options={},
|
||||
)
|
||||
chunks_no_usage: Final = list(wrapper_no_usage)
|
||||
assert len(chunks_no_usage) == 2
|
||||
assert chunks_no_usage[0].choices[0].delta.tool_calls[0].id == "call_single"
|
||||
assert getattr(chunks_no_usage[0], "usage", None) is None
|
||||
assert getattr(chunks_no_usage[1], "usage", None) is None
|
||||
assert chunks_no_usage[1].choices[0].finish_reason == "tool_calls"
|
||||
|
||||
|
||||
def test_legacy_tool_use_fallback_covered():
|
||||
from litellm.litellm_core_utils.streaming_handler import convert_generic_chunk_to_model_response_stream
|
||||
from litellm.types.llms.openai import (
|
||||
ChatCompletionToolCallChunk,
|
||||
ChatCompletionToolCallFunctionChunk,
|
||||
)
|
||||
from litellm.types.utils import GenericStreamingChunk
|
||||
|
||||
legacy_chunk: Final[GenericStreamingChunk] = GenericStreamingChunk(
|
||||
text="",
|
||||
tool_use=ChatCompletionToolCallChunk(
|
||||
id="call_legacy",
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(name="old_tool", arguments="{}"),
|
||||
index=0,
|
||||
),
|
||||
is_finished=False,
|
||||
finish_reason="",
|
||||
usage=None,
|
||||
index=0,
|
||||
)
|
||||
res_stream: Final = convert_generic_chunk_to_model_response_stream(legacy_chunk)
|
||||
assert res_stream.choices[0].delta.tool_calls is not None
|
||||
assert res_stream.choices[0].delta.tool_calls[0].id == "call_legacy"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue