mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(streaming_handler): rebase on current main with the coercion hardening
The previous commit was merged against a stale fork main, which reverted unrelated upstream work. Redo the three-way merge against BerriAI/litellm@main and keep the null-value / non-mapping guards plus their regression tests.
This commit is contained in:
parent
f1e7d661ce
commit
cf4ee17844
3 changed files with 2113 additions and 161 deletions
|
|
@ -1,21 +1,24 @@
|
|||
import base64
|
||||
import builtins
|
||||
import time
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from itertools import groupby
|
||||
from itertools import chain, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Optional, TypeAlias, TypedDict, Union, cast
|
||||
from typing import TYPE_CHECKING, Final, Optional, Protocol, TypeAlias, TypedDict, TypeVar, Union, cast
|
||||
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, Required
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionAssistantContentValue,
|
||||
ChatCompletionAudioDelta,
|
||||
)
|
||||
|
|
@ -35,12 +38,16 @@ from litellm.types.utils import (
|
|||
Function,
|
||||
FunctionCall,
|
||||
LlmProviders,
|
||||
Logprobs,
|
||||
Message,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
PromptTokensDetailsWrapper,
|
||||
ServerToolUse,
|
||||
StreamingChoices,
|
||||
TextChoices,
|
||||
TextCompletionResponse,
|
||||
UnreportedUsage,
|
||||
Usage,
|
||||
)
|
||||
from litellm.utils import print_verbose, token_counter
|
||||
|
|
@ -118,6 +125,7 @@ class _BaseChunk(TypedDict, total=False):
|
|||
created: ReadOnly[int]
|
||||
model: ReadOnly[str]
|
||||
system_fingerprint: ReadOnly[str | None]
|
||||
provider_specific_fields: ReadOnly[Mapping[str, builtins.object] | None]
|
||||
service_tier: ReadOnly[str | None]
|
||||
choices: ReadOnly[Required[Sequence[StreamingChoices]]]
|
||||
_hidden_params: ReadOnly[_ChunkHiddenParams]
|
||||
|
|
@ -418,7 +426,7 @@ class ChunkProcessor:
|
|||
if hasattr(chunk["choices"][0], "finish_reason"):
|
||||
chunk_finish_reason = chunk["choices"][0].finish_reason
|
||||
elif "finish_reason" in chunk["choices"][0]:
|
||||
chunk_finish_reason = chunk["choices"][0]["finish_reason"]
|
||||
chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"])
|
||||
if chunk_finish_reason is not None:
|
||||
finish_reason = chunk_finish_reason
|
||||
|
||||
|
|
@ -430,6 +438,11 @@ class ChunkProcessor:
|
|||
"created": created,
|
||||
"model": model,
|
||||
"system_fingerprint": system_fingerprint,
|
||||
**(
|
||||
{"provider_specific_fields": chunks[0].get("provider_specific_fields")}
|
||||
if system_fingerprint == "fp_ollama"
|
||||
else {}
|
||||
),
|
||||
**(
|
||||
MappingProxyType({"service_tier": service_tier})
|
||||
if service_tier is not None
|
||||
|
|
@ -850,7 +863,9 @@ class ChunkProcessor:
|
|||
return reasoning_tokens
|
||||
|
||||
@staticmethod
|
||||
def _extract_usage_chunk(chunk: "_UsageBearingChunk | ModelResponse | ModelResponseStream") -> Usage | None:
|
||||
def extract_usage_chunk(
|
||||
chunk: "_UsageBearingChunk | ModelResponse | ModelResponseStream | TextCompletionResponse",
|
||||
) -> Usage | None:
|
||||
usage_chunk: Usage | CompletionUsage | None = None
|
||||
if hasattr(chunk, "usage") and chunk.usage is not None:
|
||||
usage_chunk = chunk.usage
|
||||
|
|
@ -863,6 +878,8 @@ class ChunkProcessor:
|
|||
|
||||
if isinstance(usage_chunk, dict):
|
||||
return Usage(**usage_chunk)
|
||||
if isinstance(usage_chunk, UnreportedUsage):
|
||||
return None
|
||||
if usage_chunk is None or isinstance(usage_chunk, Usage):
|
||||
return usage_chunk
|
||||
return Usage(**usage_chunk.model_dump())
|
||||
|
|
@ -909,7 +926,7 @@ class ChunkProcessor:
|
|||
cost: float | None = None
|
||||
|
||||
for chunk in chunks:
|
||||
usage_chunk = self._extract_usage_chunk(chunk)
|
||||
usage_chunk = self.extract_usage_chunk(chunk)
|
||||
|
||||
if usage_chunk is not None:
|
||||
usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk)
|
||||
|
|
@ -1006,7 +1023,7 @@ class ChunkProcessor:
|
|||
values: Final = [
|
||||
value
|
||||
for chunk in chunks
|
||||
if (usage_chunk := self._extract_usage_chunk(chunk)) is not None
|
||||
if (usage_chunk := self.extract_usage_chunk(chunk)) is not None
|
||||
and isinstance(value := getattr(usage_chunk, field, None), str)
|
||||
]
|
||||
return values[-1] if values else None
|
||||
|
|
@ -1079,7 +1096,7 @@ class ChunkProcessor:
|
|||
chunks: Sequence["_UsageBearingChunk | ModelResponse"],
|
||||
model: str,
|
||||
completion_output: str,
|
||||
messages: Sequence | None = None,
|
||||
messages: Sequence[object] | None = None,
|
||||
reasoning_tokens: int | None = None,
|
||||
count_prompt_tokens: Callable[[], int] | None = None,
|
||||
) -> Usage:
|
||||
|
|
@ -1192,7 +1209,7 @@ class ChunkProcessor:
|
|||
return returned_usage
|
||||
|
||||
|
||||
def concatenate_base64_list(base64_strings: list[str]) -> str:
|
||||
def concatenate_base64_list(base64_strings: Sequence[str]) -> str:
|
||||
"""
|
||||
Concatenates a list of base64-encoded strings.
|
||||
|
||||
|
|
@ -1209,35 +1226,116 @@ def concatenate_base64_list(base64_strings: list[str]) -> str:
|
|||
return base64.b64encode(combined_bytes).decode("utf-8")
|
||||
|
||||
|
||||
def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None = None) -> TextCompletionResponse:
|
||||
id: Final = chunks[0]["id"]
|
||||
object: Final = chunks[0]["object"]
|
||||
created: Final = chunks[0]["created"]
|
||||
model: Final = chunks[0]["model"]
|
||||
system_fingerprint: Final = chunks[0].get("system_fingerprint", None)
|
||||
finish_reason: Final = chunks[-1]["choices"][0]["finish_reason"]
|
||||
logprobs: Final = chunks[-1]["choices"][0]["logprobs"]
|
||||
_CHOICE_DICTS: Final = TypeAdapter(list[dict[str, object]])
|
||||
_CHOICE_FIELD_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
content_list: Final = []
|
||||
for chunk in chunks:
|
||||
choices = chunk["choices"]
|
||||
for choice in choices:
|
||||
if choice is not None and hasattr(choice, "text") and choice.get("text") is not None:
|
||||
_choice = choice.get("text")
|
||||
content_list.append(_choice)
|
||||
|
||||
# Combine the "content" strings into a single string || combine the 'function' strings into a single string
|
||||
combined_content: Final = "".join(content_list)
|
||||
|
||||
def _text_completion_chunk_from_mapping(chunk: Mapping[str, object]) -> TextCompletionResponse:
|
||||
try:
|
||||
prompt_tokens = token_counter(model=model, messages=messages)
|
||||
except Exception: # don't allow this failing to block a complete streaming response from being returned
|
||||
print_verbose("token_counter failed, assuming prompt tokens is 0")
|
||||
prompt_tokens = 0
|
||||
completion_tokens: Final = token_counter(
|
||||
model=model,
|
||||
text=combined_content,
|
||||
count_response_tokens=True, # count_response_tokens is a Flag to tell token counter this is a response, No need to add extra tokens we do for input messages
|
||||
return TextCompletionResponse.model_validate(chunk)
|
||||
except ValidationError:
|
||||
choices: Final = _CHOICE_DICTS.validate_python(chunk.get("choices", []))
|
||||
return TextCompletionResponse.model_validate(
|
||||
{**chunk, "choices": [{**choice, "logprobs": None} for choice in choices]}
|
||||
)
|
||||
|
||||
|
||||
def _as_text_completion_chunk(
|
||||
chunk: TextCompletionResponse | ModelResponseStream | Mapping[str, object],
|
||||
) -> TextCompletionResponse:
|
||||
if isinstance(chunk, TextCompletionResponse):
|
||||
return chunk
|
||||
if isinstance(chunk, ModelResponseStream):
|
||||
return TextCompletionResponse(
|
||||
id=chunk.id,
|
||||
created=chunk.created,
|
||||
model=chunk.model,
|
||||
choices=[TextChoices(index=choice.index, finish_reason=choice.finish_reason) for choice in chunk.choices],
|
||||
usage=ChunkProcessor.extract_usage_chunk(chunk),
|
||||
)
|
||||
return _text_completion_chunk_from_mapping(chunk)
|
||||
|
||||
|
||||
_LogprobsValue: Final = TypeVar("_LogprobsValue")
|
||||
|
||||
|
||||
def _joined_or_none(fields: Iterable[Sequence[_LogprobsValue] | None]) -> list[_LogprobsValue] | None:
|
||||
sent: Final = tuple(field for field in fields if field is not None)
|
||||
return list(chain.from_iterable(sent)) if sent else None
|
||||
|
||||
|
||||
class _CarriesLogprobs(Protocol):
|
||||
@property
|
||||
def logprobs(self) -> object: ...
|
||||
|
||||
|
||||
def _sent_logprobs(choice: _CarriesLogprobs) -> object:
|
||||
return choice.logprobs
|
||||
|
||||
|
||||
def _merged_logprobs(choices: Sequence[TextChoices]) -> object:
|
||||
sent: Final = tuple(logprobs for logprobs in map(_sent_logprobs, choices) if logprobs is not None)
|
||||
parts: Final = tuple(logprobs for logprobs in sent if isinstance(logprobs, Logprobs))
|
||||
if len(parts) < len(sent):
|
||||
return sent[-1]
|
||||
if not parts:
|
||||
return None
|
||||
return Logprobs(
|
||||
text_offset=_joined_or_none(part.text_offset for part in parts),
|
||||
token_logprobs=_joined_or_none(part.token_logprobs for part in parts),
|
||||
tokens=_joined_or_none(part.tokens for part in parts),
|
||||
top_logprobs=_joined_or_none(part.top_logprobs for part in parts),
|
||||
)
|
||||
|
||||
|
||||
def _text_of(choice: TextChoices) -> str:
|
||||
return choice.text if isinstance(choice.text, str) else ""
|
||||
|
||||
|
||||
def _index_of(choice: TextChoices) -> int:
|
||||
try:
|
||||
return int(choice.index or 0)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return 0
|
||||
|
||||
|
||||
def _rebuilt_text_choice(index: int, choices: Sequence[TextChoices]) -> TextChoices:
|
||||
return TextChoices(
|
||||
text="".join(map(_text_of, choices)),
|
||||
index=index,
|
||||
logprobs=_merged_logprobs(choices),
|
||||
finish_reason=next((choice.finish_reason for choice in reversed(choices) if choice.finish_reason), None),
|
||||
)
|
||||
|
||||
|
||||
def _reported_usage(chunk: TextCompletionResponse) -> Usage | None:
|
||||
usage: Final = ChunkProcessor.extract_usage_chunk(chunk)
|
||||
return usage if usage is not None and (usage.prompt_tokens or usage.completion_tokens) else None
|
||||
|
||||
|
||||
def stream_chunk_builder_text_completion(
|
||||
chunks: Sequence[TextCompletionResponse | ModelResponseStream | Mapping[str, object]],
|
||||
messages: Sequence[AllMessageValues | Message] | None = None,
|
||||
) -> TextCompletionResponse:
|
||||
text_chunks: Final = tuple(_as_text_completion_chunk(chunk) for chunk in chunks)
|
||||
id: Final = text_chunks[0].id
|
||||
object: Final = text_chunks[0].object
|
||||
created: Final = text_chunks[0].created
|
||||
model: Final = text_chunks[0].model
|
||||
system_fingerprint: Final = text_chunks[0].get("system_fingerprint", None)
|
||||
all_choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in text_chunks))
|
||||
choices: Final = [
|
||||
_rebuilt_text_choice(index, tuple(group))
|
||||
for index, group in groupby(sorted(all_choices, key=_index_of), key=_index_of)
|
||||
] or [_rebuilt_text_choice(0, ())]
|
||||
combined_content: Final = "".join(map(_text_of, choices))
|
||||
|
||||
reported_usages: Final = tuple(reported for reported in map(_reported_usage, text_chunks) if reported is not None)
|
||||
usage: Final = ChunkProcessor(list(text_chunks), messages).calculate_usage(
|
||||
chunks=tuple(_UsageBearingChunk(usage=reported) for reported in reported_usages),
|
||||
model=model or "",
|
||||
completion_output=combined_content,
|
||||
messages=messages,
|
||||
)
|
||||
|
||||
response: Final = {
|
||||
|
|
@ -1246,19 +1344,8 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None
|
|||
"created": created,
|
||||
"model": model,
|
||||
"system_fingerprint": system_fingerprint,
|
||||
"choices": [
|
||||
{
|
||||
"text": combined_content,
|
||||
"index": 0,
|
||||
"logprobs": logprobs,
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"completion_tokens": completion_tokens,
|
||||
"total_tokens": prompt_tokens + completion_tokens,
|
||||
},
|
||||
"choices": choices,
|
||||
"usage": usage,
|
||||
}
|
||||
return TextCompletionResponse(**response)
|
||||
|
||||
|
|
@ -1266,14 +1353,27 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None
|
|||
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value})
|
||||
|
||||
|
||||
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
|
||||
def _logged_provider(logging_obj: Optional["Logging"]) -> object:
|
||||
if logging_obj is None:
|
||||
return False
|
||||
return None
|
||||
provider: Final[object] = logging_obj.model_call_details.get("custom_llm_provider")
|
||||
return provider in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
|
||||
return provider
|
||||
|
||||
|
||||
def _stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> float | None:
|
||||
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
|
||||
return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
|
||||
|
||||
|
||||
def _stream_builder_response_cost(
|
||||
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
|
||||
) -> float | None:
|
||||
if _logged_provider(logging_obj) == LlmProviders.OPENCODE_GO.value:
|
||||
# usage.opencode only survives on the chunks, the assembled Usage keeps declared fields
|
||||
reported: Final = (
|
||||
provider_reported_usage_cost(chunk_usage(chunk), LlmProviders.OPENCODE_GO.value)
|
||||
for chunk in reversed(chunks)
|
||||
)
|
||||
return next((cost for cost in reported if cost is not None), None)
|
||||
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
|
||||
if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj):
|
||||
return float(usage_cost)
|
||||
|
|
@ -1306,28 +1406,55 @@ def _stream_builder_model_map_cost(response: ModelResponse) -> float | None:
|
|||
return None
|
||||
|
||||
|
||||
def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
|
||||
response_cost: Final = _stream_builder_response_cost(response, logging_obj)
|
||||
if response_cost is None:
|
||||
return
|
||||
def _set_hidden_response_cost(response: ModelResponse, response_cost: float) -> None:
|
||||
hidden_params: Final = response._hidden_params # pyright: ignore[reportPrivateUsage] # no public accessor
|
||||
hidden_params["response_cost"] = response_cost
|
||||
|
||||
|
||||
def _set_stream_builder_response_cost(
|
||||
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
|
||||
) -> None:
|
||||
response_cost: Final = _stream_builder_response_cost(response, logging_obj, chunks)
|
||||
if response_cost is None:
|
||||
return
|
||||
_set_hidden_response_cost(response, response_cost)
|
||||
|
||||
|
||||
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
|
||||
if logging_obj is None:
|
||||
return
|
||||
if isinstance(getattr(usage, "cost", None), (int, float)):
|
||||
return
|
||||
computed_cost: Final = logging_obj.response_cost_calculator(result=response)
|
||||
if isinstance(computed_cost, (int, float)) and computed_cost > 0:
|
||||
setattr(usage, "cost", computed_cost)
|
||||
if not isinstance(computed_cost, (int, float)) or computed_cost <= 0:
|
||||
return
|
||||
setattr(usage, "cost", computed_cost)
|
||||
_set_hidden_response_cost(response, float(computed_cost))
|
||||
|
||||
|
||||
def _field_of(value: object, name: str) -> object:
|
||||
if isinstance(value, dict):
|
||||
return cast(Mapping[str, object], value).get(name) # cast-ok: chunk dicts are untyped JSON
|
||||
return getattr(value, name, None)
|
||||
|
||||
|
||||
def _delta_reasoning_items(delta: object) -> Sequence[object]:
|
||||
items: Final = _field_of(delta, "reasoning_items")
|
||||
return cast(Sequence[object], items) if isinstance(items, list) else () # cast-ok: items forwarded verbatim
|
||||
|
||||
|
||||
def _chunk_reasoning_items(chunk: object) -> Sequence[object]:
|
||||
choices: Final = _field_of(chunk, "choices")
|
||||
if not isinstance(choices, list) or not choices:
|
||||
return ()
|
||||
return _delta_reasoning_items(_field_of(cast(Sequence[object], choices)[0], "delta")) # cast-ok: untyped chunk
|
||||
|
||||
|
||||
_NON_TEXT_DELTA_FIELDS: Final = (
|
||||
"tool_calls",
|
||||
"function_call",
|
||||
"reasoning_content",
|
||||
"reasoning_items",
|
||||
"thinking_blocks",
|
||||
"annotations",
|
||||
"audio",
|
||||
|
|
@ -1405,9 +1532,14 @@ def stream_chunk_builder(
|
|||
return None
|
||||
## Route to the text completion logic
|
||||
first_chunk_with_choices: Final = next((c for c in chunks if c.get("choices")), None)
|
||||
if first_chunk_with_choices is not None and isinstance(
|
||||
first_chunk_with_choices["choices"][0], litellm.utils.TextChoices
|
||||
): # route to the text completion logic
|
||||
if first_chunk_with_choices is not None and (
|
||||
isinstance(first_chunk_with_choices["choices"][0], TextChoices)
|
||||
or (
|
||||
isinstance(first_chunk_with_choices["choices"][0], Mapping)
|
||||
and "text" in first_chunk_with_choices["choices"][0]
|
||||
and "delta" not in first_chunk_with_choices["choices"][0]
|
||||
)
|
||||
):
|
||||
return stream_chunk_builder_text_completion(chunks=chunks, messages=messages)
|
||||
|
||||
model: Final = chunks[0]["model"]
|
||||
|
|
@ -1455,7 +1587,11 @@ def stream_chunk_builder(
|
|||
break
|
||||
|
||||
_stamp_streaming_usage_cost(usage, response, logging_obj)
|
||||
_set_stream_builder_response_cost(response, logging_obj)
|
||||
_set_stream_builder_response_cost(
|
||||
response,
|
||||
logging_obj,
|
||||
chunks, # pyright: ignore[reportUnknownArgumentType] # legacy chunk list is untyped
|
||||
)
|
||||
|
||||
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
|
||||
return response
|
||||
|
|
@ -1525,6 +1661,11 @@ def stream_chunk_builder(
|
|||
reasoning_chunks
|
||||
)
|
||||
|
||||
typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream
|
||||
reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
|
||||
if reasoning_items:
|
||||
response["choices"][0]["message"]["reasoning_items"] = list(reasoning_items)
|
||||
|
||||
annotation_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
|
|
@ -1634,7 +1775,11 @@ def stream_chunk_builder(
|
|||
break
|
||||
|
||||
_stamp_streaming_usage_cost(usage, response, logging_obj)
|
||||
_set_stream_builder_response_cost(response, logging_obj)
|
||||
_set_stream_builder_response_cost(
|
||||
response,
|
||||
logging_obj,
|
||||
chunks, # pyright: ignore[reportUnknownArgumentType] # legacy chunk list is untyped
|
||||
)
|
||||
|
||||
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import base64
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
|
|
@ -15,6 +15,7 @@ from litellm.types.utils import GenericStreamingChunk
|
|||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import (
|
||||
ChunkProcessor,
|
||||
delta_content_as_str,
|
||||
stream_chunk_builder_text_completion,
|
||||
)
|
||||
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -26,6 +27,8 @@ from litellm.types.utils import (
|
|||
PromptTokensDetails,
|
||||
ServerToolUse,
|
||||
StreamingChoices,
|
||||
TextChoices,
|
||||
TextCompletionResponse,
|
||||
Usage,
|
||||
)
|
||||
|
||||
|
|
@ -2143,6 +2146,383 @@ def test_stream_chunk_builder_joins_openai_audio_output_pieces() -> None:
|
|||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reported_total", [{"total_tokens": 13}, {}])
|
||||
@pytest.mark.parametrize("trailer_choices", [[], [{"text": None, "index": 0, "logprobs": None, "finish_reason": None}]])
|
||||
def test_stream_chunk_builder_text_completion_keeps_finish_reason_and_provider_usage(
|
||||
trailer_choices: list[dict[str, object]], reported_total: dict[str, int]
|
||||
) -> None:
|
||||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder_text_completion
|
||||
from litellm.types.utils import TextCompletionResponse
|
||||
|
||||
def chunk(choices: list[dict[str, object]], **extra: object) -> TextCompletionResponse:
|
||||
return TextCompletionResponse(
|
||||
id="cmpl-1", object="text_completion", created=1, model="my-model", choices=choices, **extra
|
||||
)
|
||||
|
||||
chunks: Final = [
|
||||
chunk([{"text": "Hello", "index": 0, "logprobs": None, "finish_reason": None}]),
|
||||
chunk([{"text": " world", "index": 0, "logprobs": None, "finish_reason": None}]),
|
||||
chunk([{"text": "", "index": 0, "logprobs": None, "finish_reason": "length"}]),
|
||||
chunk(
|
||||
trailer_choices,
|
||||
usage={
|
||||
"prompt_tokens": 7,
|
||||
"completion_tokens": 6,
|
||||
**reported_total,
|
||||
"prompt_tokens_details": {"cached_tokens": 4},
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
response: Final = stream_chunk_builder_text_completion(chunks=chunks, messages=None)
|
||||
|
||||
assert response.choices[0].text == "Hello world"
|
||||
assert response.choices[0].finish_reason == "length"
|
||||
assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (7, 6, 13)
|
||||
assert response.usage.prompt_tokens_details.cached_tokens == 4
|
||||
|
||||
|
||||
def test_stream_chunk_builder_keeps_usage_dict_set_on_a_typed_text_completion_chunk() -> None:
|
||||
def chunk(choices: list[dict[str, object]]) -> TextCompletionResponse:
|
||||
return TextCompletionResponse(
|
||||
id="cmpl-1", object="text_completion", created=1, model="my-model", choices=choices
|
||||
)
|
||||
|
||||
trailer: Final = chunk([{"text": "", "index": 0, "logprobs": None, "finish_reason": "length"}])
|
||||
trailer["usage"] = {"prompt_tokens": 7, "completion_tokens": 6, "total_tokens": 13}
|
||||
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=[chunk([{"text": "Hello world", "index": 0, "logprobs": None, "finish_reason": None}]), trailer]
|
||||
)
|
||||
|
||||
assert response.choices[0].finish_reason == "length"
|
||||
assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (7, 6, 13)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("build", [litellm.stream_chunk_builder, litellm.stream_chunk_builder_text_completion])
|
||||
def test_stream_chunk_builder_rebuilds_dict_text_completion_chunks(build: Callable[..., TextCompletionResponse]) -> None:
|
||||
def chunk(choices: list[dict[str, object]], **extra: object) -> dict[str, object]:
|
||||
return {
|
||||
"id": "cmpl-1",
|
||||
"object": "text_completion",
|
||||
"created": 1,
|
||||
"model": "my-model",
|
||||
"choices": choices,
|
||||
**extra,
|
||||
}
|
||||
|
||||
chunks: Final = [
|
||||
chunk([{"text": "Hello", "index": 0, "logprobs": None, "finish_reason": None}]),
|
||||
chunk([{"text": " world", "index": 0, "logprobs": None, "finish_reason": "stop"}]),
|
||||
chunk([], usage={"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}),
|
||||
]
|
||||
|
||||
response: Final = build(chunks=chunks)
|
||||
|
||||
assert response.id == "cmpl-1"
|
||||
assert response.choices[0].text == "Hello world"
|
||||
assert response.choices[0].finish_reason == "stop"
|
||||
assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (3, 2, 5)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("choice_text", ["Hi", None])
|
||||
def test_stream_chunk_builder_keeps_dumped_chat_chunks_with_a_choice_text_on_the_chat_path(
|
||||
choice_text: str | None,
|
||||
) -> None:
|
||||
from litellm.types.utils import ModelResponseStream
|
||||
|
||||
def dumped_chunk(content: str, finish_reason: str | None) -> dict[str, object]:
|
||||
return ModelResponseStream(
|
||||
id="chatcmpl-1",
|
||||
object="chat.completion.chunk",
|
||||
created=1,
|
||||
model="my-model",
|
||||
choices=[
|
||||
{
|
||||
"index": 0,
|
||||
"text": choice_text,
|
||||
"delta": {"role": "assistant", "content": content},
|
||||
"finish_reason": finish_reason,
|
||||
"logprobs": None,
|
||||
}
|
||||
],
|
||||
).model_dump()
|
||||
|
||||
response: Final = litellm.stream_chunk_builder(chunks=[dumped_chunk("Hi", None), dumped_chunk(" there", "stop")])
|
||||
|
||||
assert isinstance(response, litellm.ModelResponse)
|
||||
assert response.choices[0].message.content == "Hi there"
|
||||
assert response.choices[0].finish_reason == "stop"
|
||||
|
||||
|
||||
def test_stream_chunk_builder_keeps_logprobs_from_every_text_completion_chunk() -> None:
|
||||
def chunk(text: str, offset: int, logprob: float, finish_reason: str | None) -> dict[str, object]:
|
||||
return {
|
||||
"id": "cmpl-1",
|
||||
"object": "text_completion",
|
||||
"created": 1,
|
||||
"model": "my-model",
|
||||
"choices": [
|
||||
{
|
||||
"text": text,
|
||||
"index": 0,
|
||||
"logprobs": {
|
||||
"tokens": [text],
|
||||
"token_logprobs": [logprob],
|
||||
"top_logprobs": [{text: logprob}],
|
||||
"text_offset": [offset],
|
||||
},
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
trailer: Final = {
|
||||
"id": "cmpl-1",
|
||||
"object": "text_completion",
|
||||
"created": 1,
|
||||
"model": "my-model",
|
||||
"choices": [],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
|
||||
}
|
||||
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=[chunk("Hello", 5, -0.1, None), chunk(" world", 10, -0.2, "length"), trailer]
|
||||
)
|
||||
|
||||
assert response.choices[0].text == "Hello world"
|
||||
assert response.choices[0].logprobs.model_dump() == {
|
||||
"tokens": ["Hello", " world"],
|
||||
"token_logprobs": [-0.1, -0.2],
|
||||
"top_logprobs": [{"Hello": -0.1}, {" world": -0.2}],
|
||||
"text_offset": [5, 10],
|
||||
}
|
||||
|
||||
|
||||
def _text_completion_chunk(*choices: dict[str, object]) -> dict[str, object]:
|
||||
return {"id": "cmpl-1", "object": "text_completion", "created": 1, "model": "my-model", "choices": list(choices)}
|
||||
|
||||
|
||||
def _text_choice(
|
||||
index: int, text: str, offset: int, top_logprobs: list[dict[str, float]] | None, finish_reason: str | None = None
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"text": text,
|
||||
"index": index,
|
||||
"logprobs": {"tokens": [text], "token_logprobs": [-0.5], "top_logprobs": top_logprobs, "text_offset": [offset]},
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
|
||||
|
||||
def test_stream_chunk_builder_rebuilds_each_text_completion_choice_by_index() -> None:
|
||||
usage_trailer: Final = {
|
||||
**_text_completion_chunk(),
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 4, "total_tokens": 5},
|
||||
}
|
||||
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=[
|
||||
_text_completion_chunk(_text_choice(0, "Hi", 5, [{"Hi": -0.5}])),
|
||||
_text_completion_chunk(_text_choice(1, "Hey", 5, [{"Hey": -0.5}])),
|
||||
_text_completion_chunk(_text_choice(0, " there", 7, [{" there": -0.5}], "stop")),
|
||||
_text_completion_chunk(_text_choice(1, " you", 8, [{" you": -0.5}], "length")),
|
||||
usage_trailer,
|
||||
]
|
||||
)
|
||||
|
||||
assert [
|
||||
(choice.index, choice.text, choice.finish_reason, choice.logprobs.tokens, choice.logprobs.text_offset)
|
||||
for choice in response.choices
|
||||
] == [(0, "Hi there", "stop", ["Hi", " there"], [5, 7]), (1, "Hey you", "length", ["Hey", " you"], [5, 8])]
|
||||
assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (1, 4)
|
||||
|
||||
|
||||
def test_stream_chunk_builder_keeps_null_text_completion_logprobs_fields_null() -> None:
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=[
|
||||
_text_completion_chunk(_text_choice(0, "Hello", 5, None)),
|
||||
_text_completion_chunk(_text_choice(0, " world", 10, None, "length")),
|
||||
]
|
||||
)
|
||||
|
||||
assert response.choices[0].logprobs.model_dump() == {
|
||||
"tokens": ["Hello", " world"],
|
||||
"token_logprobs": [-0.5, -0.5],
|
||||
"top_logprobs": None,
|
||||
"text_offset": [5, 10],
|
||||
}
|
||||
|
||||
|
||||
def test_stream_chunk_builder_keeps_a_text_completion_stream_that_ends_on_a_chat_chunk() -> None:
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=[
|
||||
TextCompletionResponse(id="cmpl-1", model="my-model", choices=[TextChoices(index=0, text="Hello")]),
|
||||
ModelResponseStream(
|
||||
id="cmpl-1",
|
||||
model="my-model",
|
||||
choices=[StreamingChoices(index=0, delta=Delta(content=" world"), finish_reason="stop")],
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
assert isinstance(response, TextCompletionResponse)
|
||||
assert [(choice.text, choice.finish_reason) for choice in response.choices] == [("Hello", "stop")]
|
||||
|
||||
|
||||
def test_stream_chunk_builder_drops_only_the_text_completion_logprobs_it_cannot_read() -> None:
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=[
|
||||
_text_completion_chunk({"text": "Hello", "index": 0, "logprobs": {"tokens": ["Hello"]}}),
|
||||
_text_completion_chunk(_text_choice(0, " world", 10, [{" world": -0.5}], "stop")),
|
||||
]
|
||||
)
|
||||
|
||||
assert response.choices[0].text == "Hello world"
|
||||
assert response.choices[0].logprobs.tokens == [" world"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"first_logprobs",
|
||||
[[-0.1], {"tokens": ["Hello"], "token_logprobs": [-0.1], "top_logprobs": None, "text_offset": [0]}],
|
||||
ids=["list", "logprobs_object"],
|
||||
)
|
||||
def test_stream_chunk_builder_keeps_text_completion_logprobs_of_another_shape(
|
||||
first_logprobs: Sequence[float] | Mapping[str, object],
|
||||
) -> None:
|
||||
def chunk(
|
||||
text: str, logprobs: Sequence[float] | Mapping[str, object], finish_reason: str | None
|
||||
) -> TextCompletionResponse:
|
||||
return TextCompletionResponse(
|
||||
id="cmpl-1",
|
||||
model="my-model",
|
||||
choices=[TextChoices(index=0, text=text, logprobs=logprobs, finish_reason=finish_reason)],
|
||||
)
|
||||
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=[chunk("Hello", first_logprobs, None), chunk(" world", [-0.2], "stop")]
|
||||
)
|
||||
|
||||
assert (response.choices[0].text, response.choices[0].logprobs) == ("Hello world", [-0.2])
|
||||
|
||||
|
||||
def test_stream_chunk_builder_keeps_the_usage_a_chat_chunk_reports_in_a_text_completion_stream() -> None:
|
||||
response: Final = stream_chunk_builder_text_completion(
|
||||
chunks=[
|
||||
TextCompletionResponse(
|
||||
id="cmpl-1", model="my-model", choices=[TextChoices(index=0, text="Hello", finish_reason="stop")]
|
||||
),
|
||||
ModelResponseStream(
|
||||
id="cmpl-1",
|
||||
model="my-model",
|
||||
choices=[],
|
||||
usage=Usage(prompt_tokens=3, completion_tokens=1, total_tokens=4),
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
assert response.usage is not None
|
||||
assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (3, 1, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("first_index", "second_index", "expected"),
|
||||
[
|
||||
(None, None, ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)),
|
||||
("0", 0, ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)),
|
||||
("0", "0", ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)),
|
||||
(float("inf"), 0, ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)),
|
||||
(
|
||||
0,
|
||||
"1",
|
||||
(
|
||||
{"index": 0, "text": "Hello", "finish_reason": None},
|
||||
{"index": 1, "text": " world", "finish_reason": "stop"},
|
||||
),
|
||||
),
|
||||
],
|
||||
ids=["none", "string_then_int", "strings", "infinity_then_int", "int_then_string"],
|
||||
)
|
||||
def test_stream_chunk_builder_rebuilds_text_completion_choice_indexes_that_are_not_ints_as_ints(
|
||||
first_index: float | str | None,
|
||||
second_index: int | str | None,
|
||||
expected: tuple[Mapping[str, object], ...],
|
||||
) -> None:
|
||||
response: Final = stream_chunk_builder_text_completion(
|
||||
chunks=[
|
||||
_text_completion_chunk({"text": "Hello", "index": first_index}),
|
||||
_text_completion_chunk({"text": " world", "index": second_index, "finish_reason": "stop"}),
|
||||
]
|
||||
)
|
||||
|
||||
assert tuple(choice.model_dump(include={"index", "text", "finish_reason"}) for choice in response.choices) == expected
|
||||
|
||||
|
||||
def test_stream_chunk_builder_text_completion_survives_uncountable_prompt() -> None:
|
||||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder_text_completion
|
||||
from litellm.types.utils import TextCompletionResponse
|
||||
|
||||
chunks: Final = [
|
||||
TextCompletionResponse(
|
||||
id="cmpl-1",
|
||||
object="text_completion",
|
||||
created=1,
|
||||
model="my-model",
|
||||
choices=[{"text": "Hello world", "index": 0, "logprobs": None, "finish_reason": "stop"}],
|
||||
)
|
||||
]
|
||||
|
||||
response: Final = stream_chunk_builder_text_completion(
|
||||
chunks=chunks, messages=[{"role": "user", "content": [{"type": "not-a-content-type"}]}]
|
||||
)
|
||||
|
||||
assert response.choices[0].text == "Hello world"
|
||||
assert response.usage.prompt_tokens == 0
|
||||
assert response.usage.completion_tokens > 0
|
||||
assert response.usage.total_tokens == response.usage.completion_tokens
|
||||
|
||||
|
||||
def test_stream_chunk_builder_keeps_reasoning_items_on_text_only_replies():
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
reasoning_items: Final[list[dict[str, object]]] = [
|
||||
{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "enc_1"}
|
||||
]
|
||||
chunks: Final = [
|
||||
ModelResponseStream(
|
||||
id="chatcmpl-rs",
|
||||
created=1,
|
||||
model="gpt-5.5",
|
||||
object="chat.completion.chunk",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
finish_reason=None,
|
||||
index=0,
|
||||
delta=Delta(content="Hello", role="assistant"),
|
||||
)
|
||||
],
|
||||
),
|
||||
ModelResponseStream(
|
||||
id="chatcmpl-rs",
|
||||
created=1,
|
||||
model="gpt-5.5",
|
||||
object="chat.completion.chunk",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
delta=Delta(content=" world", role="assistant", reasoning_items=reasoning_items),
|
||||
)
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
response: Final = stream_chunk_builder(chunks=list(chunks))
|
||||
|
||||
assert response is not None
|
||||
assert response.choices[0].message.content == "Hello world"
|
||||
assert response.choices[0].message.reasoning_items == reasoning_items
|
||||
|
||||
|
||||
def test_stream_chunk_builder_joins_content_part_list_delta():
|
||||
"""A provider streaming delta.content as a content-part list must aggregate as text."""
|
||||
chunks = [
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
Loading…
Add table
Reference in a new issue