fix(streaming_handler): rebase on current main with the coercion hardening

The previous commit was merged against a stale fork main, which reverted unrelated
upstream work. Redo the three-way merge against BerriAI/litellm@main and keep the
null-value / non-mapping guards plus their regression tests.
This commit is contained in:
inchang-ing 2026-10-27 03:32:44 +08:00
parent f1e7d661ce
commit cf4ee17844
3 changed files with 2113 additions and 161 deletions

View file

@ -1,21 +1,24 @@
import base64
import builtins
import time
from collections.abc import Callable, Iterator, Mapping, Sequence
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass, replace
from itertools import groupby
from itertools import chain, groupby
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Optional, TypeAlias, TypedDict, Union, cast
from typing import TYPE_CHECKING, Final, Optional, Protocol, TypeAlias, TypedDict, TypeVar, Union, cast
from pydantic import BaseModel
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, Required
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict
)
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantContentValue,
ChatCompletionAudioDelta,
)
@ -35,12 +38,16 @@ from litellm.types.utils import (
Function,
FunctionCall,
LlmProviders,
Logprobs,
Message,
ModelResponse,
ModelResponseStream,
PromptTokensDetailsWrapper,
ServerToolUse,
StreamingChoices,
TextChoices,
TextCompletionResponse,
UnreportedUsage,
Usage,
)
from litellm.utils import print_verbose, token_counter
@ -118,6 +125,7 @@ class _BaseChunk(TypedDict, total=False):
created: ReadOnly[int]
model: ReadOnly[str]
system_fingerprint: ReadOnly[str | None]
provider_specific_fields: ReadOnly[Mapping[str, builtins.object] | None]
service_tier: ReadOnly[str | None]
choices: ReadOnly[Required[Sequence[StreamingChoices]]]
_hidden_params: ReadOnly[_ChunkHiddenParams]
@ -418,7 +426,7 @@ class ChunkProcessor:
if hasattr(chunk["choices"][0], "finish_reason"):
chunk_finish_reason = chunk["choices"][0].finish_reason
elif "finish_reason" in chunk["choices"][0]:
chunk_finish_reason = chunk["choices"][0]["finish_reason"]
chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"])
if chunk_finish_reason is not None:
finish_reason = chunk_finish_reason
@ -430,6 +438,11 @@ class ChunkProcessor:
"created": created,
"model": model,
"system_fingerprint": system_fingerprint,
**(
{"provider_specific_fields": chunks[0].get("provider_specific_fields")}
if system_fingerprint == "fp_ollama"
else {}
),
**(
MappingProxyType({"service_tier": service_tier})
if service_tier is not None
@ -850,7 +863,9 @@ class ChunkProcessor:
return reasoning_tokens
@staticmethod
def _extract_usage_chunk(chunk: "_UsageBearingChunk | ModelResponse | ModelResponseStream") -> Usage | None:
def extract_usage_chunk(
chunk: "_UsageBearingChunk | ModelResponse | ModelResponseStream | TextCompletionResponse",
) -> Usage | None:
usage_chunk: Usage | CompletionUsage | None = None
if hasattr(chunk, "usage") and chunk.usage is not None:
usage_chunk = chunk.usage
@ -863,6 +878,8 @@ class ChunkProcessor:
if isinstance(usage_chunk, dict):
return Usage(**usage_chunk)
if isinstance(usage_chunk, UnreportedUsage):
return None
if usage_chunk is None or isinstance(usage_chunk, Usage):
return usage_chunk
return Usage(**usage_chunk.model_dump())
@ -909,7 +926,7 @@ class ChunkProcessor:
cost: float | None = None
for chunk in chunks:
usage_chunk = self._extract_usage_chunk(chunk)
usage_chunk = self.extract_usage_chunk(chunk)
if usage_chunk is not None:
usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk)
@ -1006,7 +1023,7 @@ class ChunkProcessor:
values: Final = [
value
for chunk in chunks
if (usage_chunk := self._extract_usage_chunk(chunk)) is not None
if (usage_chunk := self.extract_usage_chunk(chunk)) is not None
and isinstance(value := getattr(usage_chunk, field, None), str)
]
return values[-1] if values else None
@ -1079,7 +1096,7 @@ class ChunkProcessor:
chunks: Sequence["_UsageBearingChunk | ModelResponse"],
model: str,
completion_output: str,
messages: Sequence | None = None,
messages: Sequence[object] | None = None,
reasoning_tokens: int | None = None,
count_prompt_tokens: Callable[[], int] | None = None,
) -> Usage:
@ -1192,7 +1209,7 @@ class ChunkProcessor:
return returned_usage
def concatenate_base64_list(base64_strings: list[str]) -> str:
def concatenate_base64_list(base64_strings: Sequence[str]) -> str:
"""
Concatenates a list of base64-encoded strings.
@ -1209,35 +1226,116 @@ def concatenate_base64_list(base64_strings: list[str]) -> str:
return base64.b64encode(combined_bytes).decode("utf-8")
def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None = None) -> TextCompletionResponse:
id: Final = chunks[0]["id"]
object: Final = chunks[0]["object"]
created: Final = chunks[0]["created"]
model: Final = chunks[0]["model"]
system_fingerprint: Final = chunks[0].get("system_fingerprint", None)
finish_reason: Final = chunks[-1]["choices"][0]["finish_reason"]
logprobs: Final = chunks[-1]["choices"][0]["logprobs"]
_CHOICE_DICTS: Final = TypeAdapter(list[dict[str, object]])
_CHOICE_FIELD_VALUE: Final = TypeAdapter(object)
content_list: Final = []
for chunk in chunks:
choices = chunk["choices"]
for choice in choices:
if choice is not None and hasattr(choice, "text") and choice.get("text") is not None:
_choice = choice.get("text")
content_list.append(_choice)
# Combine the "content" strings into a single string || combine the 'function' strings into a single string
combined_content: Final = "".join(content_list)
def _text_completion_chunk_from_mapping(chunk: Mapping[str, object]) -> TextCompletionResponse:
try:
prompt_tokens = token_counter(model=model, messages=messages)
except Exception: # don't allow this failing to block a complete streaming response from being returned
print_verbose("token_counter failed, assuming prompt tokens is 0")
prompt_tokens = 0
completion_tokens: Final = token_counter(
model=model,
text=combined_content,
count_response_tokens=True, # count_response_tokens is a Flag to tell token counter this is a response, No need to add extra tokens we do for input messages
return TextCompletionResponse.model_validate(chunk)
except ValidationError:
choices: Final = _CHOICE_DICTS.validate_python(chunk.get("choices", []))
return TextCompletionResponse.model_validate(
{**chunk, "choices": [{**choice, "logprobs": None} for choice in choices]}
)
def _as_text_completion_chunk(
chunk: TextCompletionResponse | ModelResponseStream | Mapping[str, object],
) -> TextCompletionResponse:
if isinstance(chunk, TextCompletionResponse):
return chunk
if isinstance(chunk, ModelResponseStream):
return TextCompletionResponse(
id=chunk.id,
created=chunk.created,
model=chunk.model,
choices=[TextChoices(index=choice.index, finish_reason=choice.finish_reason) for choice in chunk.choices],
usage=ChunkProcessor.extract_usage_chunk(chunk),
)
return _text_completion_chunk_from_mapping(chunk)
_LogprobsValue: Final = TypeVar("_LogprobsValue")
def _joined_or_none(fields: Iterable[Sequence[_LogprobsValue] | None]) -> list[_LogprobsValue] | None:
sent: Final = tuple(field for field in fields if field is not None)
return list(chain.from_iterable(sent)) if sent else None
class _CarriesLogprobs(Protocol):
@property
def logprobs(self) -> object: ...
def _sent_logprobs(choice: _CarriesLogprobs) -> object:
return choice.logprobs
def _merged_logprobs(choices: Sequence[TextChoices]) -> object:
sent: Final = tuple(logprobs for logprobs in map(_sent_logprobs, choices) if logprobs is not None)
parts: Final = tuple(logprobs for logprobs in sent if isinstance(logprobs, Logprobs))
if len(parts) < len(sent):
return sent[-1]
if not parts:
return None
return Logprobs(
text_offset=_joined_or_none(part.text_offset for part in parts),
token_logprobs=_joined_or_none(part.token_logprobs for part in parts),
tokens=_joined_or_none(part.tokens for part in parts),
top_logprobs=_joined_or_none(part.top_logprobs for part in parts),
)
def _text_of(choice: TextChoices) -> str:
return choice.text if isinstance(choice.text, str) else ""
def _index_of(choice: TextChoices) -> int:
try:
return int(choice.index or 0)
except (TypeError, ValueError, OverflowError):
return 0
def _rebuilt_text_choice(index: int, choices: Sequence[TextChoices]) -> TextChoices:
return TextChoices(
text="".join(map(_text_of, choices)),
index=index,
logprobs=_merged_logprobs(choices),
finish_reason=next((choice.finish_reason for choice in reversed(choices) if choice.finish_reason), None),
)
def _reported_usage(chunk: TextCompletionResponse) -> Usage | None:
usage: Final = ChunkProcessor.extract_usage_chunk(chunk)
return usage if usage is not None and (usage.prompt_tokens or usage.completion_tokens) else None
def stream_chunk_builder_text_completion(
chunks: Sequence[TextCompletionResponse | ModelResponseStream | Mapping[str, object]],
messages: Sequence[AllMessageValues | Message] | None = None,
) -> TextCompletionResponse:
text_chunks: Final = tuple(_as_text_completion_chunk(chunk) for chunk in chunks)
id: Final = text_chunks[0].id
object: Final = text_chunks[0].object
created: Final = text_chunks[0].created
model: Final = text_chunks[0].model
system_fingerprint: Final = text_chunks[0].get("system_fingerprint", None)
all_choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in text_chunks))
choices: Final = [
_rebuilt_text_choice(index, tuple(group))
for index, group in groupby(sorted(all_choices, key=_index_of), key=_index_of)
] or [_rebuilt_text_choice(0, ())]
combined_content: Final = "".join(map(_text_of, choices))
reported_usages: Final = tuple(reported for reported in map(_reported_usage, text_chunks) if reported is not None)
usage: Final = ChunkProcessor(list(text_chunks), messages).calculate_usage(
chunks=tuple(_UsageBearingChunk(usage=reported) for reported in reported_usages),
model=model or "",
completion_output=combined_content,
messages=messages,
)
response: Final = {
@ -1246,19 +1344,8 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None
"created": created,
"model": model,
"system_fingerprint": system_fingerprint,
"choices": [
{
"text": combined_content,
"index": 0,
"logprobs": logprobs,
"finish_reason": finish_reason,
}
],
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
},
"choices": choices,
"usage": usage,
}
return TextCompletionResponse(**response)
@ -1266,14 +1353,27 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value})
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
def _logged_provider(logging_obj: Optional["Logging"]) -> object:
if logging_obj is None:
return False
return None
provider: Final[object] = logging_obj.model_call_details.get("custom_llm_provider")
return provider in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
return provider
def _stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> float | None:
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
def _stream_builder_response_cost(
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
) -> float | None:
if _logged_provider(logging_obj) == LlmProviders.OPENCODE_GO.value:
# usage.opencode only survives on the chunks, the assembled Usage keeps declared fields
reported: Final = (
provider_reported_usage_cost(chunk_usage(chunk), LlmProviders.OPENCODE_GO.value)
for chunk in reversed(chunks)
)
return next((cost for cost in reported if cost is not None), None)
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj):
return float(usage_cost)
@ -1306,28 +1406,55 @@ def _stream_builder_model_map_cost(response: ModelResponse) -> float | None:
return None
def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
response_cost: Final = _stream_builder_response_cost(response, logging_obj)
if response_cost is None:
return
def _set_hidden_response_cost(response: ModelResponse, response_cost: float) -> None:
hidden_params: Final = response._hidden_params # pyright: ignore[reportPrivateUsage] # no public accessor
hidden_params["response_cost"] = response_cost
def _set_stream_builder_response_cost(
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
) -> None:
response_cost: Final = _stream_builder_response_cost(response, logging_obj, chunks)
if response_cost is None:
return
_set_hidden_response_cost(response, response_cost)
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
if logging_obj is None:
return
if isinstance(getattr(usage, "cost", None), (int, float)):
return
computed_cost: Final = logging_obj.response_cost_calculator(result=response)
if isinstance(computed_cost, (int, float)) and computed_cost > 0:
setattr(usage, "cost", computed_cost)
if not isinstance(computed_cost, (int, float)) or computed_cost <= 0:
return
setattr(usage, "cost", computed_cost)
_set_hidden_response_cost(response, float(computed_cost))
def _field_of(value: object, name: str) -> object:
if isinstance(value, dict):
return cast(Mapping[str, object], value).get(name) # cast-ok: chunk dicts are untyped JSON
return getattr(value, name, None)
def _delta_reasoning_items(delta: object) -> Sequence[object]:
items: Final = _field_of(delta, "reasoning_items")
return cast(Sequence[object], items) if isinstance(items, list) else () # cast-ok: items forwarded verbatim
def _chunk_reasoning_items(chunk: object) -> Sequence[object]:
choices: Final = _field_of(chunk, "choices")
if not isinstance(choices, list) or not choices:
return ()
return _delta_reasoning_items(_field_of(cast(Sequence[object], choices)[0], "delta")) # cast-ok: untyped chunk
_NON_TEXT_DELTA_FIELDS: Final = (
"tool_calls",
"function_call",
"reasoning_content",
"reasoning_items",
"thinking_blocks",
"annotations",
"audio",
@ -1405,9 +1532,14 @@ def stream_chunk_builder(
return None
## Route to the text completion logic
first_chunk_with_choices: Final = next((c for c in chunks if c.get("choices")), None)
if first_chunk_with_choices is not None and isinstance(
first_chunk_with_choices["choices"][0], litellm.utils.TextChoices
): # route to the text completion logic
if first_chunk_with_choices is not None and (
isinstance(first_chunk_with_choices["choices"][0], TextChoices)
or (
isinstance(first_chunk_with_choices["choices"][0], Mapping)
and "text" in first_chunk_with_choices["choices"][0]
and "delta" not in first_chunk_with_choices["choices"][0]
)
):
return stream_chunk_builder_text_completion(chunks=chunks, messages=messages)
model: Final = chunks[0]["model"]
@ -1455,7 +1587,11 @@ def stream_chunk_builder(
break
_stamp_streaming_usage_cost(usage, response, logging_obj)
_set_stream_builder_response_cost(response, logging_obj)
_set_stream_builder_response_cost(
response,
logging_obj,
chunks, # pyright: ignore[reportUnknownArgumentType] # legacy chunk list is untyped
)
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
return response
@ -1525,6 +1661,11 @@ def stream_chunk_builder(
reasoning_chunks
)
typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream
reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
if reasoning_items:
response["choices"][0]["message"]["reasoning_items"] = list(reasoning_items)
annotation_chunks: Final = [
chunk
for chunk in chunks
@ -1634,7 +1775,11 @@ def stream_chunk_builder(
break
_stamp_streaming_usage_cost(usage, response, logging_obj)
_set_stream_builder_response_cost(response, logging_obj)
_set_stream_builder_response_cost(
response,
logging_obj,
chunks, # pyright: ignore[reportUnknownArgumentType] # legacy chunk list is untyped
)
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
return response

View file

@ -1,6 +1,6 @@
import base64
import json
from collections.abc import Mapping, Sequence
from collections.abc import Callable, Mapping, Sequence
from typing import Final
from unittest.mock import MagicMock
@ -15,6 +15,7 @@ from litellm.types.utils import GenericStreamingChunk
from litellm.litellm_core_utils.streaming_chunk_builder_utils import (
ChunkProcessor,
delta_content_as_str,
stream_chunk_builder_text_completion,
)
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
from litellm.types.utils import (
@ -26,6 +27,8 @@ from litellm.types.utils import (
PromptTokensDetails,
ServerToolUse,
StreamingChoices,
TextChoices,
TextCompletionResponse,
Usage,
)
@ -2143,6 +2146,383 @@ def test_stream_chunk_builder_joins_openai_audio_output_pieces() -> None:
}
@pytest.mark.parametrize("reported_total", [{"total_tokens": 13}, {}])
@pytest.mark.parametrize("trailer_choices", [[], [{"text": None, "index": 0, "logprobs": None, "finish_reason": None}]])
def test_stream_chunk_builder_text_completion_keeps_finish_reason_and_provider_usage(
trailer_choices: list[dict[str, object]], reported_total: dict[str, int]
) -> None:
from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder_text_completion
from litellm.types.utils import TextCompletionResponse
def chunk(choices: list[dict[str, object]], **extra: object) -> TextCompletionResponse:
return TextCompletionResponse(
id="cmpl-1", object="text_completion", created=1, model="my-model", choices=choices, **extra
)
chunks: Final = [
chunk([{"text": "Hello", "index": 0, "logprobs": None, "finish_reason": None}]),
chunk([{"text": " world", "index": 0, "logprobs": None, "finish_reason": None}]),
chunk([{"text": "", "index": 0, "logprobs": None, "finish_reason": "length"}]),
chunk(
trailer_choices,
usage={
"prompt_tokens": 7,
"completion_tokens": 6,
**reported_total,
"prompt_tokens_details": {"cached_tokens": 4},
},
),
]
response: Final = stream_chunk_builder_text_completion(chunks=chunks, messages=None)
assert response.choices[0].text == "Hello world"
assert response.choices[0].finish_reason == "length"
assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (7, 6, 13)
assert response.usage.prompt_tokens_details.cached_tokens == 4
def test_stream_chunk_builder_keeps_usage_dict_set_on_a_typed_text_completion_chunk() -> None:
def chunk(choices: list[dict[str, object]]) -> TextCompletionResponse:
return TextCompletionResponse(
id="cmpl-1", object="text_completion", created=1, model="my-model", choices=choices
)
trailer: Final = chunk([{"text": "", "index": 0, "logprobs": None, "finish_reason": "length"}])
trailer["usage"] = {"prompt_tokens": 7, "completion_tokens": 6, "total_tokens": 13}
response: Final = litellm.stream_chunk_builder(
chunks=[chunk([{"text": "Hello world", "index": 0, "logprobs": None, "finish_reason": None}]), trailer]
)
assert response.choices[0].finish_reason == "length"
assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (7, 6, 13)
@pytest.mark.parametrize("build", [litellm.stream_chunk_builder, litellm.stream_chunk_builder_text_completion])
def test_stream_chunk_builder_rebuilds_dict_text_completion_chunks(build: Callable[..., TextCompletionResponse]) -> None:
def chunk(choices: list[dict[str, object]], **extra: object) -> dict[str, object]:
return {
"id": "cmpl-1",
"object": "text_completion",
"created": 1,
"model": "my-model",
"choices": choices,
**extra,
}
chunks: Final = [
chunk([{"text": "Hello", "index": 0, "logprobs": None, "finish_reason": None}]),
chunk([{"text": " world", "index": 0, "logprobs": None, "finish_reason": "stop"}]),
chunk([], usage={"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}),
]
response: Final = build(chunks=chunks)
assert response.id == "cmpl-1"
assert response.choices[0].text == "Hello world"
assert response.choices[0].finish_reason == "stop"
assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (3, 2, 5)
@pytest.mark.parametrize("choice_text", ["Hi", None])
def test_stream_chunk_builder_keeps_dumped_chat_chunks_with_a_choice_text_on_the_chat_path(
choice_text: str | None,
) -> None:
from litellm.types.utils import ModelResponseStream
def dumped_chunk(content: str, finish_reason: str | None) -> dict[str, object]:
return ModelResponseStream(
id="chatcmpl-1",
object="chat.completion.chunk",
created=1,
model="my-model",
choices=[
{
"index": 0,
"text": choice_text,
"delta": {"role": "assistant", "content": content},
"finish_reason": finish_reason,
"logprobs": None,
}
],
).model_dump()
response: Final = litellm.stream_chunk_builder(chunks=[dumped_chunk("Hi", None), dumped_chunk(" there", "stop")])
assert isinstance(response, litellm.ModelResponse)
assert response.choices[0].message.content == "Hi there"
assert response.choices[0].finish_reason == "stop"
def test_stream_chunk_builder_keeps_logprobs_from_every_text_completion_chunk() -> None:
def chunk(text: str, offset: int, logprob: float, finish_reason: str | None) -> dict[str, object]:
return {
"id": "cmpl-1",
"object": "text_completion",
"created": 1,
"model": "my-model",
"choices": [
{
"text": text,
"index": 0,
"logprobs": {
"tokens": [text],
"token_logprobs": [logprob],
"top_logprobs": [{text: logprob}],
"text_offset": [offset],
},
"finish_reason": finish_reason,
}
],
}
trailer: Final = {
"id": "cmpl-1",
"object": "text_completion",
"created": 1,
"model": "my-model",
"choices": [],
"usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
}
response: Final = litellm.stream_chunk_builder(
chunks=[chunk("Hello", 5, -0.1, None), chunk(" world", 10, -0.2, "length"), trailer]
)
assert response.choices[0].text == "Hello world"
assert response.choices[0].logprobs.model_dump() == {
"tokens": ["Hello", " world"],
"token_logprobs": [-0.1, -0.2],
"top_logprobs": [{"Hello": -0.1}, {" world": -0.2}],
"text_offset": [5, 10],
}
def _text_completion_chunk(*choices: dict[str, object]) -> dict[str, object]:
return {"id": "cmpl-1", "object": "text_completion", "created": 1, "model": "my-model", "choices": list(choices)}
def _text_choice(
index: int, text: str, offset: int, top_logprobs: list[dict[str, float]] | None, finish_reason: str | None = None
) -> dict[str, object]:
return {
"text": text,
"index": index,
"logprobs": {"tokens": [text], "token_logprobs": [-0.5], "top_logprobs": top_logprobs, "text_offset": [offset]},
"finish_reason": finish_reason,
}
def test_stream_chunk_builder_rebuilds_each_text_completion_choice_by_index() -> None:
usage_trailer: Final = {
**_text_completion_chunk(),
"usage": {"prompt_tokens": 1, "completion_tokens": 4, "total_tokens": 5},
}
response: Final = litellm.stream_chunk_builder(
chunks=[
_text_completion_chunk(_text_choice(0, "Hi", 5, [{"Hi": -0.5}])),
_text_completion_chunk(_text_choice(1, "Hey", 5, [{"Hey": -0.5}])),
_text_completion_chunk(_text_choice(0, " there", 7, [{" there": -0.5}], "stop")),
_text_completion_chunk(_text_choice(1, " you", 8, [{" you": -0.5}], "length")),
usage_trailer,
]
)
assert [
(choice.index, choice.text, choice.finish_reason, choice.logprobs.tokens, choice.logprobs.text_offset)
for choice in response.choices
] == [(0, "Hi there", "stop", ["Hi", " there"], [5, 7]), (1, "Hey you", "length", ["Hey", " you"], [5, 8])]
assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (1, 4)
def test_stream_chunk_builder_keeps_null_text_completion_logprobs_fields_null() -> None:
response: Final = litellm.stream_chunk_builder(
chunks=[
_text_completion_chunk(_text_choice(0, "Hello", 5, None)),
_text_completion_chunk(_text_choice(0, " world", 10, None, "length")),
]
)
assert response.choices[0].logprobs.model_dump() == {
"tokens": ["Hello", " world"],
"token_logprobs": [-0.5, -0.5],
"top_logprobs": None,
"text_offset": [5, 10],
}
def test_stream_chunk_builder_keeps_a_text_completion_stream_that_ends_on_a_chat_chunk() -> None:
response: Final = litellm.stream_chunk_builder(
chunks=[
TextCompletionResponse(id="cmpl-1", model="my-model", choices=[TextChoices(index=0, text="Hello")]),
ModelResponseStream(
id="cmpl-1",
model="my-model",
choices=[StreamingChoices(index=0, delta=Delta(content=" world"), finish_reason="stop")],
),
]
)
assert isinstance(response, TextCompletionResponse)
assert [(choice.text, choice.finish_reason) for choice in response.choices] == [("Hello", "stop")]
def test_stream_chunk_builder_drops_only_the_text_completion_logprobs_it_cannot_read() -> None:
response: Final = litellm.stream_chunk_builder(
chunks=[
_text_completion_chunk({"text": "Hello", "index": 0, "logprobs": {"tokens": ["Hello"]}}),
_text_completion_chunk(_text_choice(0, " world", 10, [{" world": -0.5}], "stop")),
]
)
assert response.choices[0].text == "Hello world"
assert response.choices[0].logprobs.tokens == [" world"]
@pytest.mark.parametrize(
"first_logprobs",
[[-0.1], {"tokens": ["Hello"], "token_logprobs": [-0.1], "top_logprobs": None, "text_offset": [0]}],
ids=["list", "logprobs_object"],
)
def test_stream_chunk_builder_keeps_text_completion_logprobs_of_another_shape(
first_logprobs: Sequence[float] | Mapping[str, object],
) -> None:
def chunk(
text: str, logprobs: Sequence[float] | Mapping[str, object], finish_reason: str | None
) -> TextCompletionResponse:
return TextCompletionResponse(
id="cmpl-1",
model="my-model",
choices=[TextChoices(index=0, text=text, logprobs=logprobs, finish_reason=finish_reason)],
)
response: Final = litellm.stream_chunk_builder(
chunks=[chunk("Hello", first_logprobs, None), chunk(" world", [-0.2], "stop")]
)
assert (response.choices[0].text, response.choices[0].logprobs) == ("Hello world", [-0.2])
def test_stream_chunk_builder_keeps_the_usage_a_chat_chunk_reports_in_a_text_completion_stream() -> None:
response: Final = stream_chunk_builder_text_completion(
chunks=[
TextCompletionResponse(
id="cmpl-1", model="my-model", choices=[TextChoices(index=0, text="Hello", finish_reason="stop")]
),
ModelResponseStream(
id="cmpl-1",
model="my-model",
choices=[],
usage=Usage(prompt_tokens=3, completion_tokens=1, total_tokens=4),
),
]
)
assert response.usage is not None
assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (3, 1, 4)
@pytest.mark.parametrize(
("first_index", "second_index", "expected"),
[
(None, None, ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)),
("0", 0, ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)),
("0", "0", ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)),
(float("inf"), 0, ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)),
(
0,
"1",
(
{"index": 0, "text": "Hello", "finish_reason": None},
{"index": 1, "text": " world", "finish_reason": "stop"},
),
),
],
ids=["none", "string_then_int", "strings", "infinity_then_int", "int_then_string"],
)
def test_stream_chunk_builder_rebuilds_text_completion_choice_indexes_that_are_not_ints_as_ints(
first_index: float | str | None,
second_index: int | str | None,
expected: tuple[Mapping[str, object], ...],
) -> None:
response: Final = stream_chunk_builder_text_completion(
chunks=[
_text_completion_chunk({"text": "Hello", "index": first_index}),
_text_completion_chunk({"text": " world", "index": second_index, "finish_reason": "stop"}),
]
)
assert tuple(choice.model_dump(include={"index", "text", "finish_reason"}) for choice in response.choices) == expected
def test_stream_chunk_builder_text_completion_survives_uncountable_prompt() -> None:
from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder_text_completion
from litellm.types.utils import TextCompletionResponse
chunks: Final = [
TextCompletionResponse(
id="cmpl-1",
object="text_completion",
created=1,
model="my-model",
choices=[{"text": "Hello world", "index": 0, "logprobs": None, "finish_reason": "stop"}],
)
]
response: Final = stream_chunk_builder_text_completion(
chunks=chunks, messages=[{"role": "user", "content": [{"type": "not-a-content-type"}]}]
)
assert response.choices[0].text == "Hello world"
assert response.usage.prompt_tokens == 0
assert response.usage.completion_tokens > 0
assert response.usage.total_tokens == response.usage.completion_tokens
def test_stream_chunk_builder_keeps_reasoning_items_on_text_only_replies():
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
reasoning_items: Final[list[dict[str, object]]] = [
{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "enc_1"}
]
chunks: Final = [
ModelResponseStream(
id="chatcmpl-rs",
created=1,
model="gpt-5.5",
object="chat.completion.chunk",
choices=[
StreamingChoices(
finish_reason=None,
index=0,
delta=Delta(content="Hello", role="assistant"),
)
],
),
ModelResponseStream(
id="chatcmpl-rs",
created=1,
model="gpt-5.5",
object="chat.completion.chunk",
choices=[
StreamingChoices(
finish_reason="stop",
index=0,
delta=Delta(content=" world", role="assistant", reasoning_items=reasoning_items),
)
],
),
]
response: Final = stream_chunk_builder(chunks=list(chunks))
assert response is not None
assert response.choices[0].message.content == "Hello world"
assert response.choices[0].message.reasoning_items == reasoning_items
def test_stream_chunk_builder_joins_content_part_list_delta():
"""A provider streaming delta.content as a content-part list must aggregate as text."""
chunks = [

File diff suppressed because it is too large Load diff