From cf4ee1784434fcd890f2eb82504d62085506a6fa Mon Sep 17 00:00:00 2001 From: inchang-ing <291394088@qq.com> Date: Tue, 27 Oct 2026 03:32:44 +0800 Subject: [PATCH] fix(streaming_handler): rebase on current main with the coercion hardening The previous commit was merged against a stale fork main, which reverted unrelated upstream work. Redo the three-way merge against BerriAI/litellm@main and keep the null-value / non-mapping guards plus their regression tests. --- .../streaming_chunk_builder_utils.py | 273 ++- .../test_streaming_chunk_builder_utils.py | 382 +++- .../test_streaming_handler.py | 1619 ++++++++++++++++- 3 files changed, 2113 insertions(+), 161 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index c704227c2f1..fe2b129b0e2 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1,21 +1,24 @@ import base64 +import builtins import time -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass, replace -from itertools import groupby +from itertools import chain, groupby from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Optional, TypeAlias, TypedDict, Union, cast +from typing import TYPE_CHECKING, Final, Optional, Protocol, TypeAlias, TypedDict, TypeVar, Union, cast -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict ) from litellm.types.llms.openai import ( + AllMessageValues, ChatCompletionAssistantContentValue, ChatCompletionAudioDelta, ) @@ -35,12 +38,16 @@ from litellm.types.utils import ( Function, FunctionCall, LlmProviders, + Logprobs, + Message, ModelResponse, ModelResponseStream, PromptTokensDetailsWrapper, ServerToolUse, StreamingChoices, + TextChoices, TextCompletionResponse, + UnreportedUsage, Usage, ) from litellm.utils import print_verbose, token_counter @@ -118,6 +125,7 @@ class _BaseChunk(TypedDict, total=False): created: ReadOnly[int] model: ReadOnly[str] system_fingerprint: ReadOnly[str | None] + provider_specific_fields: ReadOnly[Mapping[str, builtins.object] | None] service_tier: ReadOnly[str | None] choices: ReadOnly[Required[Sequence[StreamingChoices]]] _hidden_params: ReadOnly[_ChunkHiddenParams] @@ -418,7 +426,7 @@ class ChunkProcessor: if hasattr(chunk["choices"][0], "finish_reason"): chunk_finish_reason = chunk["choices"][0].finish_reason elif "finish_reason" in chunk["choices"][0]: - chunk_finish_reason = chunk["choices"][0]["finish_reason"] + chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"]) if chunk_finish_reason is not None: finish_reason = chunk_finish_reason @@ -430,6 +438,11 @@ class ChunkProcessor: "created": created, "model": model, "system_fingerprint": system_fingerprint, + **( + {"provider_specific_fields": chunks[0].get("provider_specific_fields")} + if system_fingerprint == "fp_ollama" + else {} + ), **( MappingProxyType({"service_tier": service_tier}) if service_tier is not None @@ -850,7 +863,9 @@ class ChunkProcessor: return reasoning_tokens @staticmethod - def _extract_usage_chunk(chunk: "_UsageBearingChunk | ModelResponse | ModelResponseStream") -> Usage | None: + def extract_usage_chunk( + chunk: "_UsageBearingChunk | ModelResponse | ModelResponseStream | TextCompletionResponse", + ) -> Usage | None: usage_chunk: Usage | CompletionUsage | None = None if hasattr(chunk, "usage") and chunk.usage is not None: usage_chunk = chunk.usage @@ -863,6 +878,8 @@ class ChunkProcessor: if isinstance(usage_chunk, dict): return Usage(**usage_chunk) + if isinstance(usage_chunk, UnreportedUsage): + return None if usage_chunk is None or isinstance(usage_chunk, Usage): return usage_chunk return Usage(**usage_chunk.model_dump()) @@ -909,7 +926,7 @@ class ChunkProcessor: cost: float | None = None for chunk in chunks: - usage_chunk = self._extract_usage_chunk(chunk) + usage_chunk = self.extract_usage_chunk(chunk) if usage_chunk is not None: usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk) @@ -1006,7 +1023,7 @@ class ChunkProcessor: values: Final = [ value for chunk in chunks - if (usage_chunk := self._extract_usage_chunk(chunk)) is not None + if (usage_chunk := self.extract_usage_chunk(chunk)) is not None and isinstance(value := getattr(usage_chunk, field, None), str) ] return values[-1] if values else None @@ -1079,7 +1096,7 @@ class ChunkProcessor: chunks: Sequence["_UsageBearingChunk | ModelResponse"], model: str, completion_output: str, - messages: Sequence | None = None, + messages: Sequence[object] | None = None, reasoning_tokens: int | None = None, count_prompt_tokens: Callable[[], int] | None = None, ) -> Usage: @@ -1192,7 +1209,7 @@ class ChunkProcessor: return returned_usage -def concatenate_base64_list(base64_strings: list[str]) -> str: +def concatenate_base64_list(base64_strings: Sequence[str]) -> str: """ Concatenates a list of base64-encoded strings. @@ -1209,35 +1226,116 @@ def concatenate_base64_list(base64_strings: list[str]) -> str: return base64.b64encode(combined_bytes).decode("utf-8") -def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None = None) -> TextCompletionResponse: - id: Final = chunks[0]["id"] - object: Final = chunks[0]["object"] - created: Final = chunks[0]["created"] - model: Final = chunks[0]["model"] - system_fingerprint: Final = chunks[0].get("system_fingerprint", None) - finish_reason: Final = chunks[-1]["choices"][0]["finish_reason"] - logprobs: Final = chunks[-1]["choices"][0]["logprobs"] +_CHOICE_DICTS: Final = TypeAdapter(list[dict[str, object]]) +_CHOICE_FIELD_VALUE: Final = TypeAdapter(object) - content_list: Final = [] - for chunk in chunks: - choices = chunk["choices"] - for choice in choices: - if choice is not None and hasattr(choice, "text") and choice.get("text") is not None: - _choice = choice.get("text") - content_list.append(_choice) - - # Combine the "content" strings into a single string || combine the 'function' strings into a single string - combined_content: Final = "".join(content_list) +def _text_completion_chunk_from_mapping(chunk: Mapping[str, object]) -> TextCompletionResponse: try: - prompt_tokens = token_counter(model=model, messages=messages) - except Exception: # don't allow this failing to block a complete streaming response from being returned - print_verbose("token_counter failed, assuming prompt tokens is 0") - prompt_tokens = 0 - completion_tokens: Final = token_counter( - model=model, - text=combined_content, - count_response_tokens=True, # count_response_tokens is a Flag to tell token counter this is a response, No need to add extra tokens we do for input messages + return TextCompletionResponse.model_validate(chunk) + except ValidationError: + choices: Final = _CHOICE_DICTS.validate_python(chunk.get("choices", [])) + return TextCompletionResponse.model_validate( + {**chunk, "choices": [{**choice, "logprobs": None} for choice in choices]} + ) + + +def _as_text_completion_chunk( + chunk: TextCompletionResponse | ModelResponseStream | Mapping[str, object], +) -> TextCompletionResponse: + if isinstance(chunk, TextCompletionResponse): + return chunk + if isinstance(chunk, ModelResponseStream): + return TextCompletionResponse( + id=chunk.id, + created=chunk.created, + model=chunk.model, + choices=[TextChoices(index=choice.index, finish_reason=choice.finish_reason) for choice in chunk.choices], + usage=ChunkProcessor.extract_usage_chunk(chunk), + ) + return _text_completion_chunk_from_mapping(chunk) + + +_LogprobsValue: Final = TypeVar("_LogprobsValue") + + +def _joined_or_none(fields: Iterable[Sequence[_LogprobsValue] | None]) -> list[_LogprobsValue] | None: + sent: Final = tuple(field for field in fields if field is not None) + return list(chain.from_iterable(sent)) if sent else None + + +class _CarriesLogprobs(Protocol): + @property + def logprobs(self) -> object: ... + + +def _sent_logprobs(choice: _CarriesLogprobs) -> object: + return choice.logprobs + + +def _merged_logprobs(choices: Sequence[TextChoices]) -> object: + sent: Final = tuple(logprobs for logprobs in map(_sent_logprobs, choices) if logprobs is not None) + parts: Final = tuple(logprobs for logprobs in sent if isinstance(logprobs, Logprobs)) + if len(parts) < len(sent): + return sent[-1] + if not parts: + return None + return Logprobs( + text_offset=_joined_or_none(part.text_offset for part in parts), + token_logprobs=_joined_or_none(part.token_logprobs for part in parts), + tokens=_joined_or_none(part.tokens for part in parts), + top_logprobs=_joined_or_none(part.top_logprobs for part in parts), + ) + + +def _text_of(choice: TextChoices) -> str: + return choice.text if isinstance(choice.text, str) else "" + + +def _index_of(choice: TextChoices) -> int: + try: + return int(choice.index or 0) + except (TypeError, ValueError, OverflowError): + return 0 + + +def _rebuilt_text_choice(index: int, choices: Sequence[TextChoices]) -> TextChoices: + return TextChoices( + text="".join(map(_text_of, choices)), + index=index, + logprobs=_merged_logprobs(choices), + finish_reason=next((choice.finish_reason for choice in reversed(choices) if choice.finish_reason), None), + ) + + +def _reported_usage(chunk: TextCompletionResponse) -> Usage | None: + usage: Final = ChunkProcessor.extract_usage_chunk(chunk) + return usage if usage is not None and (usage.prompt_tokens or usage.completion_tokens) else None + + +def stream_chunk_builder_text_completion( + chunks: Sequence[TextCompletionResponse | ModelResponseStream | Mapping[str, object]], + messages: Sequence[AllMessageValues | Message] | None = None, +) -> TextCompletionResponse: + text_chunks: Final = tuple(_as_text_completion_chunk(chunk) for chunk in chunks) + id: Final = text_chunks[0].id + object: Final = text_chunks[0].object + created: Final = text_chunks[0].created + model: Final = text_chunks[0].model + system_fingerprint: Final = text_chunks[0].get("system_fingerprint", None) + all_choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in text_chunks)) + choices: Final = [ + _rebuilt_text_choice(index, tuple(group)) + for index, group in groupby(sorted(all_choices, key=_index_of), key=_index_of) + ] or [_rebuilt_text_choice(0, ())] + combined_content: Final = "".join(map(_text_of, choices)) + + reported_usages: Final = tuple(reported for reported in map(_reported_usage, text_chunks) if reported is not None) + usage: Final = ChunkProcessor(list(text_chunks), messages).calculate_usage( + chunks=tuple(_UsageBearingChunk(usage=reported) for reported in reported_usages), + model=model or "", + completion_output=combined_content, + messages=messages, ) response: Final = { @@ -1246,19 +1344,8 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None "created": created, "model": model, "system_fingerprint": system_fingerprint, - "choices": [ - { - "text": combined_content, - "index": 0, - "logprobs": logprobs, - "finish_reason": finish_reason, - } - ], - "usage": { - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "total_tokens": prompt_tokens + completion_tokens, - }, + "choices": choices, + "usage": usage, } return TextCompletionResponse(**response) @@ -1266,14 +1353,27 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) -def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool: +def _logged_provider(logging_obj: Optional["Logging"]) -> object: if logging_obj is None: - return False + return None provider: Final[object] = logging_obj.model_call_details.get("custom_llm_provider") - return provider in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS + return provider -def _stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> float | None: +def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool: + return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS + + +def _stream_builder_response_cost( + response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object] +) -> float | None: + if _logged_provider(logging_obj) == LlmProviders.OPENCODE_GO.value: + # usage.opencode only survives on the chunks, the assembled Usage keeps declared fields + reported: Final = ( + provider_reported_usage_cost(chunk_usage(chunk), LlmProviders.OPENCODE_GO.value) + for chunk in reversed(chunks) + ) + return next((cost for cost in reported if cost is not None), None) usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None) if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj): return float(usage_cost) @@ -1306,28 +1406,55 @@ def _stream_builder_model_map_cost(response: ModelResponse) -> float | None: return None -def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> None: - response_cost: Final = _stream_builder_response_cost(response, logging_obj) - if response_cost is None: - return +def _set_hidden_response_cost(response: ModelResponse, response_cost: float) -> None: hidden_params: Final = response._hidden_params # pyright: ignore[reportPrivateUsage] # no public accessor hidden_params["response_cost"] = response_cost +def _set_stream_builder_response_cost( + response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object] +) -> None: + response_cost: Final = _stream_builder_response_cost(response, logging_obj, chunks) + if response_cost is None: + return + _set_hidden_response_cost(response, response_cost) + + def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: if logging_obj is None: return if isinstance(getattr(usage, "cost", None), (int, float)): return computed_cost: Final = logging_obj.response_cost_calculator(result=response) - if isinstance(computed_cost, (int, float)) and computed_cost > 0: - setattr(usage, "cost", computed_cost) + if not isinstance(computed_cost, (int, float)) or computed_cost <= 0: + return + setattr(usage, "cost", computed_cost) + _set_hidden_response_cost(response, float(computed_cost)) + + +def _field_of(value: object, name: str) -> object: + if isinstance(value, dict): + return cast(Mapping[str, object], value).get(name) # cast-ok: chunk dicts are untyped JSON + return getattr(value, name, None) + + +def _delta_reasoning_items(delta: object) -> Sequence[object]: + items: Final = _field_of(delta, "reasoning_items") + return cast(Sequence[object], items) if isinstance(items, list) else () # cast-ok: items forwarded verbatim + + +def _chunk_reasoning_items(chunk: object) -> Sequence[object]: + choices: Final = _field_of(chunk, "choices") + if not isinstance(choices, list) or not choices: + return () + return _delta_reasoning_items(_field_of(cast(Sequence[object], choices)[0], "delta")) # cast-ok: untyped chunk _NON_TEXT_DELTA_FIELDS: Final = ( "tool_calls", "function_call", "reasoning_content", + "reasoning_items", "thinking_blocks", "annotations", "audio", @@ -1405,9 +1532,14 @@ def stream_chunk_builder( return None ## Route to the text completion logic first_chunk_with_choices: Final = next((c for c in chunks if c.get("choices")), None) - if first_chunk_with_choices is not None and isinstance( - first_chunk_with_choices["choices"][0], litellm.utils.TextChoices - ): # route to the text completion logic + if first_chunk_with_choices is not None and ( + isinstance(first_chunk_with_choices["choices"][0], TextChoices) + or ( + isinstance(first_chunk_with_choices["choices"][0], Mapping) + and "text" in first_chunk_with_choices["choices"][0] + and "delta" not in first_chunk_with_choices["choices"][0] + ) + ): return stream_chunk_builder_text_completion(chunks=chunks, messages=messages) model: Final = chunks[0]["model"] @@ -1455,7 +1587,11 @@ def stream_chunk_builder( break _stamp_streaming_usage_cost(usage, response, logging_obj) - _set_stream_builder_response_cost(response, logging_obj) + _set_stream_builder_response_cost( + response, + logging_obj, + chunks, # pyright: ignore[reportUnknownArgumentType] # legacy chunk list is untyped + ) processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) return response @@ -1525,6 +1661,11 @@ def stream_chunk_builder( reasoning_chunks ) + typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream + reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks)) + if reasoning_items: + response["choices"][0]["message"]["reasoning_items"] = list(reasoning_items) + annotation_chunks: Final = [ chunk for chunk in chunks @@ -1634,7 +1775,11 @@ def stream_chunk_builder( break _stamp_streaming_usage_cost(usage, response, logging_obj) - _set_stream_builder_response_cost(response, logging_obj) + _set_stream_builder_response_cost( + response, + logging_obj, + chunks, # pyright: ignore[reportUnknownArgumentType] # legacy chunk list is untyped + ) processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) return response diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index fb6c47a8c88..70796fe1d1d 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1,6 +1,6 @@ import base64 import json -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from typing import Final from unittest.mock import MagicMock @@ -15,6 +15,7 @@ from litellm.types.utils import GenericStreamingChunk from litellm.litellm_core_utils.streaming_chunk_builder_utils import ( ChunkProcessor, delta_content_as_str, + stream_chunk_builder_text_completion, ) from litellm.llms.anthropic.chat.handler import ModelResponseIterator from litellm.types.utils import ( @@ -26,6 +27,8 @@ from litellm.types.utils import ( PromptTokensDetails, ServerToolUse, StreamingChoices, + TextChoices, + TextCompletionResponse, Usage, ) @@ -2143,6 +2146,383 @@ def test_stream_chunk_builder_joins_openai_audio_output_pieces() -> None: } +@pytest.mark.parametrize("reported_total", [{"total_tokens": 13}, {}]) +@pytest.mark.parametrize("trailer_choices", [[], [{"text": None, "index": 0, "logprobs": None, "finish_reason": None}]]) +def test_stream_chunk_builder_text_completion_keeps_finish_reason_and_provider_usage( + trailer_choices: list[dict[str, object]], reported_total: dict[str, int] +) -> None: + from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder_text_completion + from litellm.types.utils import TextCompletionResponse + + def chunk(choices: list[dict[str, object]], **extra: object) -> TextCompletionResponse: + return TextCompletionResponse( + id="cmpl-1", object="text_completion", created=1, model="my-model", choices=choices, **extra + ) + + chunks: Final = [ + chunk([{"text": "Hello", "index": 0, "logprobs": None, "finish_reason": None}]), + chunk([{"text": " world", "index": 0, "logprobs": None, "finish_reason": None}]), + chunk([{"text": "", "index": 0, "logprobs": None, "finish_reason": "length"}]), + chunk( + trailer_choices, + usage={ + "prompt_tokens": 7, + "completion_tokens": 6, + **reported_total, + "prompt_tokens_details": {"cached_tokens": 4}, + }, + ), + ] + + response: Final = stream_chunk_builder_text_completion(chunks=chunks, messages=None) + + assert response.choices[0].text == "Hello world" + assert response.choices[0].finish_reason == "length" + assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (7, 6, 13) + assert response.usage.prompt_tokens_details.cached_tokens == 4 + + +def test_stream_chunk_builder_keeps_usage_dict_set_on_a_typed_text_completion_chunk() -> None: + def chunk(choices: list[dict[str, object]]) -> TextCompletionResponse: + return TextCompletionResponse( + id="cmpl-1", object="text_completion", created=1, model="my-model", choices=choices + ) + + trailer: Final = chunk([{"text": "", "index": 0, "logprobs": None, "finish_reason": "length"}]) + trailer["usage"] = {"prompt_tokens": 7, "completion_tokens": 6, "total_tokens": 13} + + response: Final = litellm.stream_chunk_builder( + chunks=[chunk([{"text": "Hello world", "index": 0, "logprobs": None, "finish_reason": None}]), trailer] + ) + + assert response.choices[0].finish_reason == "length" + assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (7, 6, 13) + + +@pytest.mark.parametrize("build", [litellm.stream_chunk_builder, litellm.stream_chunk_builder_text_completion]) +def test_stream_chunk_builder_rebuilds_dict_text_completion_chunks(build: Callable[..., TextCompletionResponse]) -> None: + def chunk(choices: list[dict[str, object]], **extra: object) -> dict[str, object]: + return { + "id": "cmpl-1", + "object": "text_completion", + "created": 1, + "model": "my-model", + "choices": choices, + **extra, + } + + chunks: Final = [ + chunk([{"text": "Hello", "index": 0, "logprobs": None, "finish_reason": None}]), + chunk([{"text": " world", "index": 0, "logprobs": None, "finish_reason": "stop"}]), + chunk([], usage={"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}), + ] + + response: Final = build(chunks=chunks) + + assert response.id == "cmpl-1" + assert response.choices[0].text == "Hello world" + assert response.choices[0].finish_reason == "stop" + assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (3, 2, 5) + + +@pytest.mark.parametrize("choice_text", ["Hi", None]) +def test_stream_chunk_builder_keeps_dumped_chat_chunks_with_a_choice_text_on_the_chat_path( + choice_text: str | None, +) -> None: + from litellm.types.utils import ModelResponseStream + + def dumped_chunk(content: str, finish_reason: str | None) -> dict[str, object]: + return ModelResponseStream( + id="chatcmpl-1", + object="chat.completion.chunk", + created=1, + model="my-model", + choices=[ + { + "index": 0, + "text": choice_text, + "delta": {"role": "assistant", "content": content}, + "finish_reason": finish_reason, + "logprobs": None, + } + ], + ).model_dump() + + response: Final = litellm.stream_chunk_builder(chunks=[dumped_chunk("Hi", None), dumped_chunk(" there", "stop")]) + + assert isinstance(response, litellm.ModelResponse) + assert response.choices[0].message.content == "Hi there" + assert response.choices[0].finish_reason == "stop" + + +def test_stream_chunk_builder_keeps_logprobs_from_every_text_completion_chunk() -> None: + def chunk(text: str, offset: int, logprob: float, finish_reason: str | None) -> dict[str, object]: + return { + "id": "cmpl-1", + "object": "text_completion", + "created": 1, + "model": "my-model", + "choices": [ + { + "text": text, + "index": 0, + "logprobs": { + "tokens": [text], + "token_logprobs": [logprob], + "top_logprobs": [{text: logprob}], + "text_offset": [offset], + }, + "finish_reason": finish_reason, + } + ], + } + + trailer: Final = { + "id": "cmpl-1", + "object": "text_completion", + "created": 1, + "model": "my-model", + "choices": [], + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + + response: Final = litellm.stream_chunk_builder( + chunks=[chunk("Hello", 5, -0.1, None), chunk(" world", 10, -0.2, "length"), trailer] + ) + + assert response.choices[0].text == "Hello world" + assert response.choices[0].logprobs.model_dump() == { + "tokens": ["Hello", " world"], + "token_logprobs": [-0.1, -0.2], + "top_logprobs": [{"Hello": -0.1}, {" world": -0.2}], + "text_offset": [5, 10], + } + + +def _text_completion_chunk(*choices: dict[str, object]) -> dict[str, object]: + return {"id": "cmpl-1", "object": "text_completion", "created": 1, "model": "my-model", "choices": list(choices)} + + +def _text_choice( + index: int, text: str, offset: int, top_logprobs: list[dict[str, float]] | None, finish_reason: str | None = None +) -> dict[str, object]: + return { + "text": text, + "index": index, + "logprobs": {"tokens": [text], "token_logprobs": [-0.5], "top_logprobs": top_logprobs, "text_offset": [offset]}, + "finish_reason": finish_reason, + } + + +def test_stream_chunk_builder_rebuilds_each_text_completion_choice_by_index() -> None: + usage_trailer: Final = { + **_text_completion_chunk(), + "usage": {"prompt_tokens": 1, "completion_tokens": 4, "total_tokens": 5}, + } + + response: Final = litellm.stream_chunk_builder( + chunks=[ + _text_completion_chunk(_text_choice(0, "Hi", 5, [{"Hi": -0.5}])), + _text_completion_chunk(_text_choice(1, "Hey", 5, [{"Hey": -0.5}])), + _text_completion_chunk(_text_choice(0, " there", 7, [{" there": -0.5}], "stop")), + _text_completion_chunk(_text_choice(1, " you", 8, [{" you": -0.5}], "length")), + usage_trailer, + ] + ) + + assert [ + (choice.index, choice.text, choice.finish_reason, choice.logprobs.tokens, choice.logprobs.text_offset) + for choice in response.choices + ] == [(0, "Hi there", "stop", ["Hi", " there"], [5, 7]), (1, "Hey you", "length", ["Hey", " you"], [5, 8])] + assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (1, 4) + + +def test_stream_chunk_builder_keeps_null_text_completion_logprobs_fields_null() -> None: + response: Final = litellm.stream_chunk_builder( + chunks=[ + _text_completion_chunk(_text_choice(0, "Hello", 5, None)), + _text_completion_chunk(_text_choice(0, " world", 10, None, "length")), + ] + ) + + assert response.choices[0].logprobs.model_dump() == { + "tokens": ["Hello", " world"], + "token_logprobs": [-0.5, -0.5], + "top_logprobs": None, + "text_offset": [5, 10], + } + + +def test_stream_chunk_builder_keeps_a_text_completion_stream_that_ends_on_a_chat_chunk() -> None: + response: Final = litellm.stream_chunk_builder( + chunks=[ + TextCompletionResponse(id="cmpl-1", model="my-model", choices=[TextChoices(index=0, text="Hello")]), + ModelResponseStream( + id="cmpl-1", + model="my-model", + choices=[StreamingChoices(index=0, delta=Delta(content=" world"), finish_reason="stop")], + ), + ] + ) + + assert isinstance(response, TextCompletionResponse) + assert [(choice.text, choice.finish_reason) for choice in response.choices] == [("Hello", "stop")] + + +def test_stream_chunk_builder_drops_only_the_text_completion_logprobs_it_cannot_read() -> None: + response: Final = litellm.stream_chunk_builder( + chunks=[ + _text_completion_chunk({"text": "Hello", "index": 0, "logprobs": {"tokens": ["Hello"]}}), + _text_completion_chunk(_text_choice(0, " world", 10, [{" world": -0.5}], "stop")), + ] + ) + + assert response.choices[0].text == "Hello world" + assert response.choices[0].logprobs.tokens == [" world"] + + +@pytest.mark.parametrize( + "first_logprobs", + [[-0.1], {"tokens": ["Hello"], "token_logprobs": [-0.1], "top_logprobs": None, "text_offset": [0]}], + ids=["list", "logprobs_object"], +) +def test_stream_chunk_builder_keeps_text_completion_logprobs_of_another_shape( + first_logprobs: Sequence[float] | Mapping[str, object], +) -> None: + def chunk( + text: str, logprobs: Sequence[float] | Mapping[str, object], finish_reason: str | None + ) -> TextCompletionResponse: + return TextCompletionResponse( + id="cmpl-1", + model="my-model", + choices=[TextChoices(index=0, text=text, logprobs=logprobs, finish_reason=finish_reason)], + ) + + response: Final = litellm.stream_chunk_builder( + chunks=[chunk("Hello", first_logprobs, None), chunk(" world", [-0.2], "stop")] + ) + + assert (response.choices[0].text, response.choices[0].logprobs) == ("Hello world", [-0.2]) + + +def test_stream_chunk_builder_keeps_the_usage_a_chat_chunk_reports_in_a_text_completion_stream() -> None: + response: Final = stream_chunk_builder_text_completion( + chunks=[ + TextCompletionResponse( + id="cmpl-1", model="my-model", choices=[TextChoices(index=0, text="Hello", finish_reason="stop")] + ), + ModelResponseStream( + id="cmpl-1", + model="my-model", + choices=[], + usage=Usage(prompt_tokens=3, completion_tokens=1, total_tokens=4), + ), + ] + ) + + assert response.usage is not None + assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (3, 1, 4) + + +@pytest.mark.parametrize( + ("first_index", "second_index", "expected"), + [ + (None, None, ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)), + ("0", 0, ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)), + ("0", "0", ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)), + (float("inf"), 0, ({"index": 0, "text": "Hello world", "finish_reason": "stop"},)), + ( + 0, + "1", + ( + {"index": 0, "text": "Hello", "finish_reason": None}, + {"index": 1, "text": " world", "finish_reason": "stop"}, + ), + ), + ], + ids=["none", "string_then_int", "strings", "infinity_then_int", "int_then_string"], +) +def test_stream_chunk_builder_rebuilds_text_completion_choice_indexes_that_are_not_ints_as_ints( + first_index: float | str | None, + second_index: int | str | None, + expected: tuple[Mapping[str, object], ...], +) -> None: + response: Final = stream_chunk_builder_text_completion( + chunks=[ + _text_completion_chunk({"text": "Hello", "index": first_index}), + _text_completion_chunk({"text": " world", "index": second_index, "finish_reason": "stop"}), + ] + ) + + assert tuple(choice.model_dump(include={"index", "text", "finish_reason"}) for choice in response.choices) == expected + + +def test_stream_chunk_builder_text_completion_survives_uncountable_prompt() -> None: + from litellm.litellm_core_utils.streaming_chunk_builder_utils import stream_chunk_builder_text_completion + from litellm.types.utils import TextCompletionResponse + + chunks: Final = [ + TextCompletionResponse( + id="cmpl-1", + object="text_completion", + created=1, + model="my-model", + choices=[{"text": "Hello world", "index": 0, "logprobs": None, "finish_reason": "stop"}], + ) + ] + + response: Final = stream_chunk_builder_text_completion( + chunks=chunks, messages=[{"role": "user", "content": [{"type": "not-a-content-type"}]}] + ) + + assert response.choices[0].text == "Hello world" + assert response.usage.prompt_tokens == 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens == response.usage.completion_tokens + + +def test_stream_chunk_builder_keeps_reasoning_items_on_text_only_replies(): + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + reasoning_items: Final[list[dict[str, object]]] = [ + {"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "enc_1"} + ] + chunks: Final = [ + ModelResponseStream( + id="chatcmpl-rs", + created=1, + model="gpt-5.5", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content="Hello", role="assistant"), + ) + ], + ), + ModelResponseStream( + id="chatcmpl-rs", + created=1, + model="gpt-5.5", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(content=" world", role="assistant", reasoning_items=reasoning_items), + ) + ], + ), + ] + + response: Final = stream_chunk_builder(chunks=list(chunks)) + + assert response is not None + assert response.choices[0].message.content == "Hello world" + assert response.choices[0].message.reasoning_items == reasoning_items + + def test_stream_chunk_builder_joins_content_part_list_delta(): """A provider streaming delta.content as a content-part list must aggregate as text.""" chunks = [ diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 59fcecd9041..0ee22a12fac 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -1,14 +1,26 @@ import asyncio +import datetime import json +import queue +import sys +import threading import time -from typing import Final, NoReturn, Optional +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import CodeType, FrameType, MappingProxyType +from typing import AsyncIterator, Callable, Final, Iterator, Literal, NoReturn, Optional, cast from unittest.mock import AsyncMock, MagicMock, Mock, patch +import httpx import pytest - +from openai.types.chat import ChatCompletionChunk +from pydantic import BaseModel, TypeAdapter import litellm from litellm import verbose_logger from litellm._logging import session_id_var, trace_id_var +from litellm.exceptions import MidStreamFallbackError +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.streaming_handler import ( CustomStreamWrapper, @@ -16,6 +28,7 @@ from litellm.litellm_core_utils.streaming_handler import ( _ProviderChunkParsed, ) from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.responses.mcp.chat_completions_handler import _add_mcp_metadata_to_response from litellm.types.utils import ( CompletionTokensDetailsWrapper, Delta, @@ -23,6 +36,7 @@ from litellm.types.utils import ( ModelResponseStream, PromptTokensDetailsWrapper, StreamingChoices, + StandardLoggingPayload, Usage, ) from litellm.utils import ModelResponseListIterator @@ -30,6 +44,94 @@ from litellm import completion from typing import Tuple +@pytest.mark.parametrize("async_stream", [False, True]) +@pytest.mark.parametrize("transport", ["sdk", "http_handler"]) +@pytest.mark.parametrize("fingerprint", ["fp_ollama", "fp_openai", None]) +async def test_ollama_stream_ids_match_usage_and_spend_logs( + async_stream: bool, + transport: str, + fingerprint: str | None, +) -> None: + from datetime import datetime + + from openai.types.chat import ChatCompletionChunk + + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import convert_to_model_response_object + from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + parser: Final = OpenAIChatCompletionStreamingHandler(streaming_response=None, sync_stream=True) + bodies: Final = [ + {"choices": [{"index": 0, "delta": {"role": "assistant", "content": "h"}, "finish_reason": None}]}, + {"choices": [{"index": 0, "delta": {"content": "i"}, "finish_reason": None}]}, + {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {"choices": [], "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}}, + ] + raw_chunks: Final = [ + { + "id": "chatcmpl-0", + "object": "chat.completion.chunk", + "model": "gpt-oss:20b", + "created": 1, + "system_fingerprint": fingerprint, + **body, + } + for body in bodies + ] + chunks: Final = [ + ChatCompletionChunk(**body) if transport == "sdk" else parser.chunk_parser(body) for body in raw_chunks + ] + streams: Final = [ + CustomStreamWrapper( + completion_stream=ModelResponseListIterator(model_responses=chunks), + model="gpt-oss:20b", + custom_llm_provider="openai", + stream_options={"include_usage": True}, + logging_obj=Logging( + model="gpt-oss:20b", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="acompletion", + start_time=1.0, + litellm_call_id="same-client-call-id", + function_id="test", + ), + ) + for _ in range(2) + ] + received_streams: Final = ( + [chunk async for chunk in streams[0]] if async_stream else list(streams[0]), + [chunk async for chunk in streams[1]] if async_stream else list(streams[1]), + ) + for received in received_streams: + assembled: Final = litellm.stream_chunk_builder(chunks=received) + assert assembled.choices[0].message.content == "hi" + assert assembled.usage.total_tokens == 6 + assert len({chunk.id for chunk in received}) == 1 + assert received[-1].usage.total_tokens == 6 + assert assembled.id == received[0].id + reparsed: Final = convert_to_model_response_object( + response_object=assembled.model_dump(), + model_response_object=ModelResponse(), + ) + assert reparsed.id == assembled.id + logged: Final = get_logging_payload( + kwargs={"call_type": "acompletion", "litellm_call_id": "same-client-call-id", "response_cost": 0.01}, + response_obj=assembled, + start_time=datetime(2025, 1, 1), + end_time=datetime(2025, 1, 1), + ) + assert logged["request_id"] == assembled.id + if fingerprint == "fp_ollama": + assert assembled.id != "chatcmpl-0" + assert received[0].provider_specific_fields == {"original_response_id": "chatcmpl-0"} + assert assembled.provider_specific_fields == {"original_response_id": "chatcmpl-0"} + else: + assert assembled.id == "chatcmpl-0" + assert len({chunks[0].id for chunks in received_streams}) == (2 if fingerprint == "fp_ollama" else 1) + assert all(chunk.id == "chatcmpl-0" for chunk in chunks) + + @pytest.fixture def initialized_custom_stream_wrapper() -> CustomStreamWrapper: streaming_handler = CustomStreamWrapper( @@ -502,6 +604,91 @@ async def test_streaming_handler_with_usage( assert chunk_has_usage +def _usage_test_logging( + caller_stream_options: dict[str, bool] | None, stream_usage_injected: bool = False +) -> Logging: + logging_obj: Final = Logging( + model="hosted_vllm/m", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="completion", + start_time=1_790_000_000.0, + litellm_call_id="12345", + function_id="1245", + kwargs={"_litellm_stream_usage_injected": True} if stream_usage_injected else {}, + ) + logging_obj.update_environment_variables( + litellm_params={}, optional_params={}, model="hosted_vllm/m", stream_options=caller_stream_options + ) + return logging_obj + + +@pytest.mark.parametrize( + ("caller_stream_options", "stream_usage_injected", "usage_chunk_choice_count"), + [ + ({"include_usage": True}, False, 0), + (None, False, 1), + ({"include_usage": True}, True, 1), + ], + ids=["caller-asked", "provider-handler-injected", "proxy-injected"], +) +@pytest.mark.parametrize("plain_iterator", [False, True]) +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_usage_chunk_has_empty_choices_only_for_callers_that_asked( + sync_mode: bool, + plain_iterator: bool, + caller_stream_options: dict[str, bool] | None, + stream_usage_injected: bool, + usage_chunk_choice_count: int, +): + usage: Final = Usage(prompt_tokens=5, completion_tokens=1, total_tokens=6) + provider_chunks: Final = [ + ModelResponseStream(id="c1", choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="ok"))]), + ModelResponseStream(id="c1", choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")]), + ModelResponseStream(id="c1", choices=[], provider_specific_fields={"citations": ["a"]}), + ModelResponseStream(id="c1", choices=[], usage=usage), + ] + wrapper: Final = CustomStreamWrapper( + completion_stream=( + iter(provider_chunks) if plain_iterator else ModelResponseListIterator(model_responses=provider_chunks) + ), + model="hosted_vllm/m", + custom_llm_provider="hosted_vllm", + logging_obj=_usage_test_logging(caller_stream_options, stream_usage_injected), + stream_options={"include_usage": True}, + ) + + chunks: Final = list(wrapper) if sync_mode else [chunk async for chunk in wrapper] + + assert [len(chunk.choices) for chunk in chunks] == [1, 1, 0, usage_chunk_choice_count] + assert chunks[2].provider_specific_fields == {"citations": ["a"]} + assert [getattr(chunk, "usage", None) is not None for chunk in chunks] == [False, False, False, True] + assert chunks[-1].usage.total_tokens == 6 + + +@pytest.mark.asyncio +async def test_plain_iterator_stream_keeps_usage_on_its_chunk_when_usage_was_not_requested(): + usage: Final = Usage(prompt_tokens=5, completion_tokens=1, total_tokens=6) + provider_chunks: Final = [ + ModelResponseStream(id="c1", choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="ok"))]), + ModelResponseStream( + id="c1", choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")], usage=usage + ), + ] + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(provider_chunks), + model="hosted_vllm/m", + custom_llm_provider="hosted_vllm", + logging_obj=_usage_test_logging(None), + ) + + chunks: Final = [chunk async for chunk in wrapper] + + assert [chunk.choices[0].finish_reason for chunk in chunks] == [None, "stop"] + assert chunks[-1].usage.total_tokens == 6 + + @pytest.mark.parametrize("sync_mode", [False]) @pytest.mark.asyncio @pytest.mark.flaky(reruns=3) @@ -548,9 +735,11 @@ async def test_streaming_with_usage_and_logging(sync_mode: bool): mock_callback, "async_log_stream_event" ) as mock_async_log_stream_event, ): + existing_tasks = asyncio.all_tasks() await test_streaming_handler_with_usage( sync_mode=sync_mode, final_usage_block=final_usage_block ) + spawned_tasks = asyncio.all_tasks() - existing_tasks if sync_mode: time.sleep(1) mock_log_success_event.assert_called_once() @@ -560,7 +749,7 @@ async def test_streaming_with_usage_and_logging(sync_mode: bool): == final_usage_block ) else: - await asyncio.sleep(1) + await asyncio.wait_for(asyncio.gather(*spawned_tasks, return_exceptions=True), timeout=5) mock_async_log_success_event.assert_called_once() # mock_async_log_stream_event.assert_called() assert ( @@ -729,6 +918,42 @@ def test_set_response_id_propagation_valid_to_invalid( assert initialized_custom_stream_wrapper.response_id == "first-valid-id" +@pytest.mark.parametrize( + "last_delta", + [ + Delta(content="Hello"), + Delta(provider_specific_fields={"stop_reason": "refusal"}), + ], + ids=["content", "provider_specific_fields"], +) +def test_logged_response_keeps_received_model_id_when_the_finish_chunk_is_synthesized( + logging_obj: Logging, last_delta: Delta +): + upstream_id: Final = "chatcmpl-upstream-id" + wrapper: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator( + model_responses=[ + ModelResponseStream( + id=upstream_id, + model="gpt-4o-mini", + choices=[StreamingChoices(index=0, delta=last_delta, finish_reason="content_filter")], + ) + ] + ), + model="gpt-4o-mini", + logging_obj=logging_obj, + custom_llm_provider="openai", + ) + + emitted: Final = list(wrapper) + logged: Final = litellm.stream_chunk_builder(chunks=wrapper.chunks) + + assert len(emitted) == 2 + assert [chunk._hidden_params.get("received_model_id") for chunk in emitted] == [upstream_id, upstream_id] + assert logged is not None + assert logged._hidden_params.get("received_model_id") == upstream_id + + @pytest.mark.asyncio async def test_streaming_completion_start_time(logging_obj: Logging): """Test that the start time is set correctly""" @@ -750,10 +975,12 @@ async def test_streaming_completion_start_time(logging_obj: Logging): logging_obj=logging_obj, ) + existing_tasks = asyncio.all_tasks() async for chunk in response: print(chunk) - await asyncio.sleep(2) + spawned_tasks = asyncio.all_tasks() - existing_tasks + await asyncio.wait_for(asyncio.gather(*spawned_tasks, return_exceptions=True), timeout=5) assert logging_obj.model_call_details["completion_start_time"] is not None assert ( @@ -890,6 +1117,121 @@ async def test_bridged_stream_mid_stream_fallback_error_is_rebuilt_around_the_pr assert excinfo.value.generated_content == "" +@pytest.mark.asyncio +@pytest.mark.parametrize("merge_reasoning", [True, False], ids=["merged_into_content", "kept_as_reasoning"]) +async def test_mid_stream_break_after_reasoning_disqualifies_continuation(logging_obj: Logging, merge_reasoning: bool): + import httpx + + from litellm.exceptions import MidStreamFallbackError + + class _ReasoningThenCut: + def __init__(self) -> None: + self.chunks = iter( + ( + ModelResponseStream(choices=[{"index": 0, "delta": {"reasoning_content": "Let me think"}}]), + ModelResponseStream(choices=[{"index": 0, "delta": {"content": "The answer"}}]), + ) + ) + + def __aiter__(self) -> "_ReasoningThenCut": + return self + + async def __anext__(self) -> ModelResponseStream: + next_chunk: Final = next(self.chunks, None) + if next_chunk is None: + raise httpx.ReadTimeout("connection dropped") + return next_chunk + + response: Final = CustomStreamWrapper( + completion_stream=_ReasoningThenCut(), model="deepseek-reasoner", logging_obj=logging_obj + ) + response.merge_reasoning_content_in_choices = merge_reasoning + + with pytest.raises(MidStreamFallbackError) as excinfo: + async for _ in response: + pass + + assert excinfo.value.is_pre_first_chunk is False + assert excinfo.value.emitted_disqualifying_content is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "outer_delta,inner_flag,expected", + [ + ({"tool_calls": [{"index": 0}]}, False, True), + ({"content": "Let me check."}, True, True), + ({"content": "Let me check."}, False, False), + ], + ids=["bridge_wrapper_sent_a_tool_call", "inner_stream_sent_a_tool_call", "plain_text_only"], +) +async def test_bridged_stream_rebuild_keeps_the_continuation_disqualifier( + logging_obj: Logging, outer_delta: dict[str, object], inner_flag: bool, expected: bool +): + from litellm.exceptions import InternalServerError, MidStreamFallbackError + + provider_error: Final = InternalServerError(message="upstream 500", llm_provider="azure", model="gpt-6.1-sol") + inner_error: Final = MidStreamFallbackError( + message=str(provider_error), + model="gpt-6.1-sol", + llm_provider="azure", + original_exception=provider_error, + is_pre_first_chunk=False, + emitted_disqualifying_content=inner_flag, + ) + + async def _raise_inner_error(**kwargs: object) -> NoReturn: + raise inner_error + + response: Final = CustomStreamWrapper( + completion_stream=None, + model="gpt-6.1-sol", + logging_obj=logging_obj, + custom_llm_provider="azure", + make_call=_raise_inner_error, + ) + response._accumulate_streamed_delta(outer_delta) + + with pytest.raises(MidStreamFallbackError) as excinfo: + await response.__anext__() + + assert excinfo.value.emitted_disqualifying_content is expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_streaming_rate_limit_midstream_fallback_keeps_provider_headers(logging_obj: Logging, sync_mode: bool): + """MidStreamFallbackError keeps the provider headers the non-streaming RateLimitError carries.""" + from litellm.exceptions import MidStreamFallbackError + from litellm.llms.vertex_ai.common_utils import VertexAIError + + provider_headers: Final = {"retry-after": "3", "x-request-id": "req-abc123"} + + def _raise_rate_limit(**kwargs: object) -> NoReturn: + raise VertexAIError(status_code=429, message="Resource exhausted.", headers=provider_headers) + + async def _araise_rate_limit(**kwargs: object) -> NoReturn: + _raise_rate_limit(**kwargs) + + response: Final = CustomStreamWrapper( + completion_stream=None, + model="gemini-3-flash-preview", + logging_obj=logging_obj, + custom_llm_provider="vertex_ai_beta", + make_call=_raise_rate_limit if sync_mode else _araise_rate_limit, + ) + + async def _read_first_chunk() -> object: + return next(response) if sync_mode else await response.__anext__() + + with pytest.raises(MidStreamFallbackError) as excinfo: + await _read_first_chunk() + + assert excinfo.value.status_code == 429 + assert excinfo.value.litellm_response_headers.get("retry-after") == "3" + assert excinfo.value.litellm_response_headers.get("x-request-id") == "req-abc123" + + def test_sync_streaming_bad_request_not_midstream(logging_obj: Logging): """Ensure __next__ raises BadRequestError (400) directly, not MidStreamFallbackError. @@ -1135,7 +1477,187 @@ async def test_async_streaming_read_timeout_triggers_midstream_fallback( await response.__anext__() assert excinfo.value.is_pre_first_chunk is True - assert isinstance(excinfo.value.original_exception, Exception) + assert isinstance(excinfo.value.original_exception, litellm.Timeout) + + +def _raw_read_timeout() -> NoReturn: + raise httpx.ReadTimeout("") + + +def _bridged_read_timeout() -> NoReturn: + raise litellm.Timeout( + message="APITimeoutError - Request timed out. Error_str: ReadTimeout", model="glm", llm_provider="hosted_vllm" + ) from httpx.ReadTimeout("") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stall", [_raw_read_timeout, _bridged_read_timeout], ids=["raw", "responses_bridge"]) +async def test_async_streaming_read_timeout_after_first_chunk_surfaces_timeout( + logging_obj: Logging, stall: Callable[[], NoReturn] +): + from litellm.exceptions import MidStreamFallbackError + + class _OneChunkThenStall: + def __init__(self) -> None: + self.sent = False + + def __aiter__(self) -> "_OneChunkThenStall": + return self + + async def __anext__(self) -> ModelResponseStream: + if self.sent: + stall() + self.sent = True + return ModelResponseStream(choices=[{"index": 0, "delta": {"content": "hello"}}]) + + response: Final = CustomStreamWrapper( + completion_stream=_OneChunkThenStall(), + model="glm", + logging_obj=logging_obj, + custom_llm_provider="hosted_vllm", + ) + + with pytest.raises(MidStreamFallbackError) as excinfo: + async for _ in response: + pass + + assert excinfo.value.is_pre_first_chunk is False + assert isinstance(excinfo.value.original_exception, litellm.Timeout) + assert excinfo.value.original_exception.status_code == 408 + + +_AZURE_DEPLOYMENT_API_BASE: Final = "https://mid-stream-debug.openai.azure.com" +_AZURE_DEPLOYMENT_REQUEST_URL: Final = ( + f"{_AZURE_DEPLOYMENT_API_BASE}/openai/deployments/gpt-5.4-mini/chat/completions?api-version=2025-04-01-preview" +) +_MID_STREAM_START_TIME: Final = 1_790_000_000.0 + + +def _azure_deployment_logging_obj(logged_api_base: str | None, routed_api_base: str | None) -> Logging: + logging_obj: Final = Logging( + model="gpt-5.4-mini", + messages=[{"role": "user", "content": "Count from 1 to 40"}], + stream=True, + call_type="acompletion", + start_time=_MID_STREAM_START_TIME, + litellm_call_id="mid-stream-debug-info", + function_id="mid-stream-debug-info", + ) + logging_obj.update_environment_variables( + model="gpt-5.4-mini", + optional_params={}, + litellm_params=get_litellm_params( + api_base=logged_api_base, + custom_llm_provider="azure", + metadata={ + "model_group": "gpt-5.4-mini-azure", + "deployment": "azure/gpt-5.4-mini", + "api_base": routed_api_base, + }, + ), + ) + return logging_obj + + +def _content_then_failure(failure: Literal["transport_error", "unparseable_chunk"]) -> Iterator[object]: + yield ModelResponseStream( + model="gpt-5.4-mini-2026-03-17", + choices=[StreamingChoices(index=0, delta=Delta(content="1", role="assistant"))], + ) + if failure == "transport_error": + raise httpx.ReadError("Response payload is not completed") + yield object() + + +async def _as_async(chunks: Iterator[object]) -> AsyncIterator[object]: + for chunk in chunks: + yield chunk + + +async def _drain(wrapper: CustomStreamWrapper, sync_mode: bool) -> None: + if sync_mode: + for _ in wrapper: + pass + return + async for _ in wrapper: + pass + + +async def _mid_stream_debug_info( + logged_api_base: str | None, + routed_api_base: str | None, + failure: Literal["transport_error", "unparseable_chunk"], + sync_mode: bool, +) -> str: + chunks: Final = _content_then_failure(failure) + wrapper: Final = CustomStreamWrapper( + completion_stream=chunks if sync_mode else _as_async(chunks), + model="gpt-5.4-mini-2026-03-17", + logging_obj=_azure_deployment_logging_obj(logged_api_base, routed_api_base), + custom_llm_provider="azure", + ) + with pytest.raises(MidStreamFallbackError) as excinfo: + await _drain(wrapper, sync_mode) + assert excinfo.value.is_pre_first_chunk is False + debug_info: Final = getattr(excinfo.value.original_exception, "litellm_debug_info", None) + assert isinstance(debug_info, str) + return debug_info + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize("failure", ["transport_error", "unparseable_chunk"]) +@pytest.mark.parametrize( + ("logged_api_base", "routed_api_base"), + [ + (_AZURE_DEPLOYMENT_REQUEST_URL, _AZURE_DEPLOYMENT_API_BASE), + (_AZURE_DEPLOYMENT_API_BASE, None), + (_AZURE_DEPLOYMENT_API_BASE, ""), + ], + ids=["routed", "sdk", "empty_routed"], +) +async def test_mid_stream_error_reports_the_deployment_it_was_sent_to( + logged_api_base: str, + routed_api_base: str | None, + failure: Literal["transport_error", "unparseable_chunk"], + sync_mode: bool, +): + debug_info: Final = await _mid_stream_debug_info(logged_api_base, routed_api_base, failure, sync_mode) + + assert f"API Base: `{_AZURE_DEPLOYMENT_API_BASE}`" in debug_info + assert "model_group: `gpt-5.4-mini-azure`" in debug_info + assert "deployment: `azure/gpt-5.4-mini`" in debug_info + assert "Count from 1 to 40" not in debug_info + assert "api.openai.com" not in debug_info + + +@pytest.mark.asyncio +async def test_mid_stream_error_leaves_out_an_api_base_it_cannot_resolve(): + debug_info: Final = await _mid_stream_debug_info(None, None, "transport_error", sync_mode=False) + + assert "model_group: `gpt-5.4-mini-azure`" in debug_info + assert "API Base:" not in debug_info + + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "routed_api_base", + [ + f"{_AZURE_DEPLOYMENT_API_BASE}?subscription-key=deployment-secret-value", + f"{_AZURE_DEPLOYMENT_API_BASE}?code=deployment-secret-value", + _AZURE_DEPLOYMENT_API_BASE.replace("https://", "https://qa-user:deployment-secret-value@"), + ], + ids=["subscription_key", "function_code", "userinfo"], +) +async def test_mid_stream_error_keeps_credentials_in_the_api_base_out(routed_api_base: str): + debug_info: Final = await _mid_stream_debug_info( + _AZURE_DEPLOYMENT_REQUEST_URL, routed_api_base, "transport_error", sync_mode=False + ) + + assert f"API Base: `{_AZURE_DEPLOYMENT_API_BASE}`" in debug_info + assert "deployment-secret" not in debug_info + def test_streaming_handler_with_created_time_propagation( @@ -2298,6 +2820,123 @@ def test_raise_on_model_repetition_tolerates_empty_choices( wrapper.raise_on_model_repetition() +_STUCK_CHUNK: Final = "the model is stuck on this" +_STUCK_PROVIDER_CHUNK_CAP: Final = 10 * litellm.REPEATED_STREAMING_CHUNK_LIMIT + + +class _StuckProvider: + def __init__(self) -> None: + self.closed_early = threading.Event() + self.chunks_sent = 0 + + def _next_chunk(self) -> ModelResponseStream: + self.chunks_sent += 1 + return _make_chunk(_STUCK_CHUNK) + + def sync_stream(self) -> Iterator[ModelResponseStream]: + try: + yield from (self._next_chunk() for _ in range(_STUCK_PROVIDER_CHUNK_CAP)) + except GeneratorExit: + self.closed_early.set() + raise + + async def async_stream(self) -> AsyncIterator[ModelResponseStream]: + try: + for _ in range(_STUCK_PROVIDER_CHUNK_CAP): + yield self._next_chunk() + except GeneratorExit: + self.closed_early.set() + raise + + +class _StuckStreamFailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.failures: Final = queue.SimpleQueue[str]() + + def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + self.failures.put(str(cast(Mapping[str, object], kwargs)["exception"])) + + async def async_log_failure_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + self.failures.put(str(cast(Mapping[str, object], kwargs)["exception"])) + + +def _stuck_stream_wrapper( + provider_stream: Iterator[ModelResponseStream] | AsyncIterator[ModelResponseStream], + call_type: Literal["completion", "acompletion"], + recorder: _StuckStreamFailureRecorder, +) -> CustomStreamWrapper: + logging_obj: Final = Logging( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type=call_type, + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id=f"stuck-stream-{call_type}", + function_id=f"stuck-stream-{call_type}", + dynamic_failure_callbacks=[recorder], + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="test-model", + optional_params={}, + litellm_params=get_litellm_params(custom_llm_provider="openai", acompletion=call_type == "acompletion"), + ) + return CustomStreamWrapper( + completion_stream=provider_stream, + model="test-model", + logging_obj=logging_obj, + custom_llm_provider="openai", + ) + + +@pytest.mark.asyncio +async def test_async_stream_stuck_on_one_chunk_closes_the_provider_stream_and_logs_one_failure(): + """https://github.com/BerriAI/litellm/issues/19428 and https://github.com/BerriAI/litellm/issues/13786""" + provider: Final = _StuckProvider() + recorder: Final = _StuckStreamFailureRecorder() + wrapper: Final = _stuck_stream_wrapper(provider.async_stream(), "acompletion", recorder) + + with pytest.raises(Exception, match="repeating the same chunk"): + async for _ in wrapper: + pass + assert provider.closed_early.is_set() + assert provider.chunks_sent <= litellm.REPEATED_STREAMING_CHUNK_LIMIT + 1 + chunks_sent_at_failure: Final = provider.chunks_sent + + with pytest.raises(StopAsyncIteration): + await wrapper.__anext__() + await wrapper.wait_for_failure_logging(timeout=5) + + assert provider.chunks_sent == chunks_sent_at_failure + assert "repeating the same chunk" in recorder.failures.get_nowait() + assert recorder.failures.empty() + + +def test_sync_stream_stuck_on_one_chunk_closes_the_provider_stream_and_logs_one_failure(): + """Sync counterpart of the async test above""" + provider: Final = _StuckProvider() + recorder: Final = _StuckStreamFailureRecorder() + wrapper: Final = _stuck_stream_wrapper(provider.sync_stream(), "completion", recorder) + + with pytest.raises(Exception, match="repeating the same chunk"): + for _ in wrapper: + pass + assert provider.closed_early.is_set() + assert provider.chunks_sent <= litellm.REPEATED_STREAMING_CHUNK_LIMIT + 1 + chunks_sent_at_failure: Final = provider.chunks_sent + assert "repeating the same chunk" in recorder.failures.get(timeout=5) + + with pytest.raises(StopIteration): + next(wrapper) + + assert provider.chunks_sent == chunks_sent_at_failure + with pytest.raises(queue.Empty): + recorder.failures.get(timeout=0.5) + + def test_usage_chunk_after_finish_reason_updates_hidden_params(logging_obj): """ Test that provider-reported usage from a post-finish_reason chunk @@ -2387,6 +3026,238 @@ def test_usage_chunk_after_finish_reason_updates_hidden_params(logging_obj): ), f"Expected completion_tokens=135 from provider, got {hidden_usage.completion_tokens}" +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize("ends_with_finish_chunk", [True, False]) +@pytest.mark.asyncio +async def test_stream_without_provider_usage_falls_back_to_the_token_estimate( + sync_mode: bool, ends_with_finish_chunk: bool +) -> None: + model: Final = "gpt-4o" + messages: Final = [{"role": "user", "content": "Write a long sentence about a fox. " * 30}] + text: Final = "The quick brown fox jumps over the lazy dog. " * 20 + content_chunk: Final = ModelResponseStream( + id="chatcmpl-no-usage", + created=1741037890, + model=model, + choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content=text))], + ) + finish_chunk: Final = ModelResponseStream( + id="chatcmpl-no-usage", + created=1741037890, + model=model, + choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")], + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator( + model_responses=[content_chunk, finish_chunk] if ends_with_finish_chunk else [content_chunk] + ), + model=model, + custom_llm_provider="openai", + logging_obj=Logging( + model=model, + messages=messages, + stream=True, + call_type="completion", + start_time=datetime.datetime(2025, 3, 3, 21, 38, 10), + litellm_call_id="no-usage-call", + function_id="no-usage-fn", + ), + stream_options=None, + ) + + yielded: Final = ( + [(chunk, chunk._hidden_params.get("usage")) for chunk in wrapper] + if sync_mode + else [(chunk, chunk._hidden_params.get("usage")) async for chunk in wrapper] + ) + collected: Final = [chunk for chunk, _ in yielded] + + expected: Final = ( + litellm.token_counter(model=model, messages=messages), + litellm.token_counter(model=model, text=text, count_response_tokens=True), + ) + assembled: Final = litellm.stream_chunk_builder(chunks=collected, messages=messages) + assert assembled is not None + assert (assembled.usage.prompt_tokens, assembled.usage.completion_tokens) == expected + last_chunk, usage_when_yielded = yielded[-1] + assert usage_when_yielded is not None + usage_after_drain: Final = last_chunk._hidden_params["usage"] + assert (usage_after_drain.prompt_tokens, usage_after_drain.completion_tokens) == expected + + rates: Final = litellm.model_cost[model] + spend: Final = expected[0] * rates["input_cost_per_token"] + expected[1] * rates["output_cost_per_token"] + assert litellm.completion_cost(completion_response=assembled) == pytest.approx(spend) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_stream_with_usage_after_the_finish_chunk_bills_only_the_reported_usage(sync_mode: bool) -> None: + model: Final = "gpt-4o" + messages: Final = [{"role": "user", "content": "Write a long sentence about a fox. " * 30}] + text: Final = "The quick brown fox jumps over the lazy dog. " * 20 + reported: Final = (6, 20) + chunks: Final = [ + ModelResponseStream(id="cmpl-late-usage", created=1741037890, model=model, choices=choices, usage=usage) + for choices, usage in ( + ([StreamingChoices(index=0, delta=Delta(role="assistant", content=text))], None), + ([StreamingChoices(index=0, delta=Delta(content=""), finish_reason="length")], None), + ([], Usage(prompt_tokens=reported[0], completion_tokens=reported[1], total_tokens=sum(reported))), + ) + ] + wrapper: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator(model_responses=chunks), + model=model, + custom_llm_provider="openai", + logging_obj=Logging( + model=model, + messages=messages, + stream=True, + call_type="completion", + start_time=datetime.datetime(2025, 3, 3, 21, 38, 10), + litellm_call_id="late-usage-call", + function_id="late-usage-fn", + ), + stream_options=None, + ) + + collected: Final = [chunk for chunk in wrapper] if sync_mode else [chunk async for chunk in wrapper] + + usage_after_drain: Final = collected[-1]._hidden_params["usage"] + assert (usage_after_drain.prompt_tokens, usage_after_drain.completion_tokens) == reported + assembled: Final = litellm.stream_chunk_builder(chunks=collected, messages=messages) + assert assembled is not None + rates: Final = litellm.model_cost[model] + spend: Final = reported[0] * rates["input_cost_per_token"] + reported[1] * rates["output_cost_per_token"] + assert litellm.completion_cost(completion_response=assembled) == pytest.approx(spend) + + +class _LoggedResponseRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.responses: list[object] = [] + self.logged: Final = threading.Event() + + def log_success_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + self.responses.append(response_obj) + self.logged.set() + + async def async_log_success_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + self.responses.append(response_obj) + self.logged.set() + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_stream_without_provider_usage_logs_the_estimate_with_the_final_chunk_mcp_metadata( + sync_mode: bool, +) -> None: + model: Final = "gpt-4o" + messages: Final = [{"role": "user", "content": "Write a long sentence about a fox. " * 30}] + text: Final = "The quick brown fox jumps over the lazy dog. " * 20 + tool_calls: Final = [{"name": "lookup_fox", "arguments": "{}"}] + recorder: Final = _LoggedResponseRecorder() + logging_obj: Final = Logging( + model=model, + messages=messages, + stream=True, + call_type="completion" if sync_mode else "acompletion", + start_time=datetime.datetime(2025, 3, 3, 21, 38, 10), + litellm_call_id="mcp-no-usage-call", + function_id="mcp-no-usage-fn", + dynamic_success_callbacks=[recorder], + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model=model, user=None, optional_params={}, litellm_params={"litellm_call_id": "mcp-no-usage-call"} + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator( + model_responses=[ + ModelResponseStream(id="chatcmpl-mcp", created=1741037890, model=model, choices=choices) + for choices in ( + [StreamingChoices(index=0, delta=Delta(role="assistant", content=text))], + [StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")], + ) + ] + ), + model=model, + custom_llm_provider="openai", + logging_obj=logging_obj, + stream_options=None, + ) + _add_mcp_metadata_to_response(wrapper, openai_tools=None, tool_calls=tool_calls) + + _ = [chunk for chunk in wrapper] if sync_mode else [chunk async for chunk in wrapper] + + assert await asyncio.to_thread(recorder.logged.wait, 5) + assert len(recorder.responses) == 1 + logged: Final = recorder.responses[0] + assert isinstance(logged, ModelResponse) + assert logged.choices[0].message.provider_specific_fields == {"mcp_tool_calls": tool_calls} + assert (logged.usage.prompt_tokens, logged.usage.completion_tokens) == ( + litellm.token_counter(model=model, messages=messages), + litellm.token_counter(model=model, text=text, count_response_tokens=True), + ) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize("ends_with_finish_chunk", [True, False]) +@pytest.mark.parametrize("reported_tokens", [(30, 12), (0, 0)]) +@pytest.mark.asyncio +async def test_stream_with_provider_usage_hands_that_usage_to_the_last_chunk( + logging_obj: Logging, sync_mode: bool, ends_with_finish_chunk: bool, reported_tokens: tuple[int, int] +) -> None: + model: Final = "gpt-4o" + reported: Final = Usage( + prompt_tokens=reported_tokens[0], + completion_tokens=reported_tokens[1], + total_tokens=sum(reported_tokens), + ) + content_chunk: Final = ModelResponseStream( + id="chatcmpl-with-usage", + created=1741037890, + model=model, + choices=[ + StreamingChoices( + index=0, + delta=Delta(role="assistant", content="Hello there"), + finish_reason=None if ends_with_finish_chunk else "stop", + ) + ], + **({} if ends_with_finish_chunk else {"usage": reported}), + ) + finish_chunk: Final = ModelResponseStream( + id="chatcmpl-with-usage", + created=1741037890, + model=model, + choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")], + usage=reported, + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator( + model_responses=[content_chunk, finish_chunk] if ends_with_finish_chunk else [content_chunk] + ), + model=model, + custom_llm_provider="openai", + logging_obj=logging_obj, + stream_options=None, + ) + + yielded: Final = ( + [(chunk, chunk._hidden_params.get("usage")) for chunk in wrapper] + if sync_mode + else [(chunk, chunk._hidden_params.get("usage")) async for chunk in wrapper] + ) + + last_chunk, usage_when_yielded = yielded[-1] + assert usage_when_yielded is not None + assert (usage_when_yielded.prompt_tokens, usage_when_yielded.completion_tokens) == reported_tokens + usage_after_drain: Final = last_chunk._hidden_params["usage"] + assert (usage_after_drain.prompt_tokens, usage_after_drain.completion_tokens) == reported_tokens + + @pytest.mark.asyncio async def test_custom_stream_wrapper_aclose(): """Test that aclose() delegates to the underlying completion_stream's aclose()""" @@ -2952,15 +3823,19 @@ async def test_custom_stream_wrapper_anext_does_not_block_event_loop_for_sync_it """ class BlockingIterator: - def __init__(self, chunks, delay_s: float): + def __init__(self, chunks: list[ModelResponseStream]) -> None: self._it = iter(chunks) - self._delay_s = delay_s + self.entered = threading.Event() + self.release = threading.Event() + self.returned = threading.Event() - def __iter__(self): + def __iter__(self) -> "BlockingIterator": return self - def __next__(self): - time.sleep(self._delay_s) # simulate blocking I/O + def __next__(self) -> ModelResponseStream: + self.entered.set() + self.release.wait(timeout=5) + self.returned.set() return next(self._it) test_chunk = ModelResponseStream( @@ -2988,34 +3863,23 @@ async def test_custom_stream_wrapper_anext_does_not_block_event_loop_for_sync_it usage=None, ) - # Delay is intentionally > the wait_for timeout used to detect event loop blocking. + blocking_iterator = BlockingIterator([test_chunk]) wrapper = CustomStreamWrapper( - completion_stream=BlockingIterator([test_chunk], delay_s=0.3), + completion_stream=blocking_iterator, model="test-model", logging_obj=logging_obj, custom_llm_provider="cached_response", ) - tick_event = asyncio.Event() + anext_task = asyncio.create_task(wrapper.__anext__()) + try: + assert await asyncio.to_thread(blocking_iterator.entered.wait, 5) + assert not blocking_iterator.returned.is_set(), "Event loop was blocked by the sync iterator" + finally: + blocking_iterator.release.set() - async def background_tick(): - await asyncio.sleep(0.05) - tick_event.set() - - # Run the two coroutines concurrently and measure wall time. - # If __anext__ blocks the event loop, background_tick can't run and the gather - # takes the full 0.3 s delay; if non-blocking both finish within ~0.35 s total. - start = asyncio.get_event_loop().time() - - out, _ = await asyncio.gather( - wrapper.__anext__(), - background_tick(), - ) - - elapsed = asyncio.get_event_loop().time() - start + out = await anext_task assert isinstance(out, ModelResponseStream) - # background_tick sleeps 0.05 s; total must finish well under 2 × 0.3 s - assert elapsed < 0.5, f"Event loop was likely blocked (elapsed={elapsed:.2f}s)" @pytest.mark.asyncio @@ -3519,6 +4383,68 @@ def test_record_partial_usage_for_failure_stashes_usage_and_cost(): assert isinstance(logging_obj.model_call_details["response_cost"], float) +class _FailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.failures: tuple[tuple[int, int], ...] = () + self.logged: asyncio.Event = asyncio.Event() + + async def async_log_failure_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + payload: Final = TypeAdapter(StandardLoggingPayload).validate_python(kwargs["standard_logging_object"]) + self.failures = (*self.failures, (payload["prompt_tokens"], payload["completion_tokens"])) + self.logged.set() + + +@pytest.mark.asyncio +async def test_log_stream_failure_bills_the_dropped_attempt_even_after_the_router_discards_its_usage() -> None: + """The dropped attempt's failure log is queued, and the router discards the stashed usage as it opens the + fallback hop before that log runs. The log must still bill what the dropped attempt consumed, and the shared + logging object must not be marked as already failed, since the hop's own failure still has to reach the + integrations.""" + recorder: Final = _FailureRecorder() + logging_obj: Final = Logging( + model="gpt-5.6", + messages=[{"role": "user", "content": "Hey"}], + stream=True, + call_type="completion", + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id="partial-usage-copy", + function_id="1245", + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.model_call_details["custom_llm_provider"] = "openai" + wrapper: Final = CustomStreamWrapper( + completion_stream=None, + model="gpt-5.6", + logging_obj=logging_obj, + custom_llm_provider="openai", + ) + wrapper.chunks = [ + ModelResponseStream( + id="chatcmpl-partial-copy", + created=1742056047, + model="gpt-5.6", + object="chat.completion.chunk", + choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="The", role="assistant"))], + usage=Usage(prompt_tokens=30, completion_tokens=1, total_tokens=31), + ) + ] + + with pytest.raises(MidStreamFallbackError): + wrapper._log_stream_failure_and_raise(httpx.ReadError("connection dropped")) + logging_obj.carry_failed_attempt_usage() + await asyncio.wait_for(recorder.logged.wait(), timeout=10) + + assert recorder.failures == ((30, 1),) + assert logging_obj.model_call_details.get("has_logged_async_failure") is not True + + def test_record_partial_usage_for_failure_noop_without_chunks(): """With no chunks delivered there is nothing billed to recover, so the failure stash must stay absent and not force a zero-usage row. @@ -4646,6 +5572,472 @@ def test_handle_stream_fallback_error_restores_context_only_after_exception_mapp session_id_var.set("") +@dataclass(frozen=True, slots=True) +class _ProfiledCall: + code: CodeType + arguments: Mapping[str, object] + + +def _chunk_creator_calls( + wrapper: CustomStreamWrapper, + chunks: Sequence[ChatCompletionChunk | ModelResponseStream], + codes: frozenset[CodeType], +) -> tuple[tuple[ModelResponseStream | None, ...], tuple[_ProfiledCall, ...]]: + calls: Final[list[_ProfiledCall]] = [] + + def record(frame: FrameType, event: str, _arg: object) -> None: + if event == "call" and frame.f_code in codes: + arguments: Final = dict(cast(Mapping[str, object], frame.f_locals)) + calls.append(_ProfiledCall(code=frame.f_code, arguments=MappingProxyType(arguments))) + + previous_profiler: Final = sys.getprofile() + sys.setprofile(record) + try: + outputs: Final = tuple(wrapper.chunk_creator(chunk=chunk) for chunk in chunks) + finally: + sys.setprofile(previous_profiler) + return outputs, tuple(calls) + + +def test_openai_chunk_creator_skips_full_chunk_dump_and_delta_rebuild( + logging_obj: Logging, +) -> None: + chunks: Final[list[ChatCompletionChunk]] = [ + ChatCompletionChunk( + id="chunk-1", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 0, "delta": {"role": "assistant", "content": "hello"}, "finish_reason": None}], + ), + ChatCompletionChunk( + id="chunk-2", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 0, "delta": {"content": " world"}, "finish_reason": None}], + service_tier="default", + citations=["https://a"], + custom_chunk_attribute="custom value", + ), + ] + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="gpt-4o", + custom_llm_provider="openai", + logging_obj=logging_obj, + ) + chunk_model_dump: Final = ChatCompletionChunk.model_dump.__code__ + delta_init: Final = Delta.__init__.__code__ + streaming_choices_init: Final = StreamingChoices.__init__.__code__ + + outputs, calls = _chunk_creator_calls( + wrapper, chunks, frozenset({chunk_model_dump, delta_init, streaming_choices_init}) + ) + emitted_chunks: Final = tuple(output for output in outputs if output is not None) + choice_dumps: Final = tuple( + call + for call in calls + if call.code is chunk_model_dump and not isinstance(call.arguments["self"], ChatCompletionChunk) + ) + full_chunk_dumps: Final = tuple( + call + for call in calls + if call.code is chunk_model_dump + and isinstance(call.arguments["self"], ChatCompletionChunk) + and call.arguments["include"] is None + ) + delta_constructions: Final = tuple(call for call in calls if call.code is delta_init) + streaming_choice_constructions: Final = tuple(call for call in calls if call.code is streaming_choices_init) + + assert len(emitted_chunks) == 2 + assert full_chunk_dumps == () + assert choice_dumps == () + assert len(delta_constructions) <= len(emitted_chunks) * 2 + assert len(streaming_choice_constructions) <= len(emitted_chunks) + assert [chunk.choices[0].delta.content for chunk in emitted_chunks] == ["hello", " world"] + assert emitted_chunks[0].choices[0].delta.role == "assistant" + assert emitted_chunks[1].choices[0].delta.role is None + assert getattr(emitted_chunks[1], "citations") == ["https://a"] + assert getattr(emitted_chunks[1], "custom_chunk_attribute") == "custom value" + + assert getattr(emitted_chunks[1], "service_tier") == "default" + chunk2_delta: Final = emitted_chunks[1].model_dump()["choices"][0]["delta"] + assert "provider_specific_fields" in chunk2_delta + assert chunk2_delta["provider_specific_fields"] is None + + finish_output: Final = wrapper.chunk_creator( + chunk=ChatCompletionChunk( + id="chunk-3", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}], + ) + ) + + assert finish_output is not None + assert finish_output.choices[0].finish_reason == "stop" + assert finish_output.choices[0].delta.content is None + + +def _openai_chunk(chunk_id: str, delta: Mapping[str, object]) -> ChatCompletionChunk: + return ChatCompletionChunk( + id=chunk_id, + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 0, "delta": delta, "finish_reason": None}], + ) + + +def _missed_attributes(wrapper: CustomStreamWrapper, chunks: Sequence[ChatCompletionChunk]) -> tuple[str, ...]: + attribute_lookup: Final = BaseModel.__getattr__.__code__ + misses: Final[list[str]] = [] + + def record_miss(frame: FrameType, event: str, arg: object) -> Callable[[FrameType, str, object], object]: + if event == "exception" and isinstance(arg, tuple) and isinstance(arg[1], AttributeError): + misses.append(str(frame.f_locals["item"])) + return record_miss + + def trace_lookups(frame: FrameType, event: str, _arg: object) -> Callable[[FrameType, str, object], object] | None: + return record_miss if event == "call" and frame.f_code is attribute_lookup else None + + previous_tracer: Final = sys.gettrace() + sys.settrace(trace_lookups) + try: + tuple(wrapper.chunk_creator(chunk=chunk) for chunk in chunks) + finally: + sys.settrace(previous_tracer) + return tuple(misses) + + +def test_openai_chunk_creator_builds_one_delta_and_misses_no_attribute_per_later_chunk( + logging_obj: Logging, +) -> None: + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="gpt-4o", + custom_llm_provider="openai", + logging_obj=logging_obj, + ) + wrapper.chunk_creator(chunk=_openai_chunk("chunk-1", {"role": "assistant", "content": ""})) + plain_chunks: Final = ( + _openai_chunk("chunk-2", {"content": "hello"}), + _openai_chunk("chunk-3", {"content": " world"}), + ) + nested_chunk: Final = _openai_chunk( + "chunk-4", {"content": "!", "reasoning_details": [{"type": "reasoning.text", "text": "thought"}]} + ) + delta_init: Final = Delta.__init__.__code__ + streaming_choices_init: Final = StreamingChoices.__init__.__code__ + + outputs, calls = _chunk_creator_calls(wrapper, plain_chunks, frozenset({delta_init, streaming_choices_init})) + + assert sum(call.code is delta_init for call in calls) == len(plain_chunks) + assert sum(call.code is streaming_choices_init for call in calls) == len(plain_chunks) + assert [output.choices[0].delta.content if output else None for output in outputs] == ["hello", " world"] + assert _missed_attributes(wrapper, (*plain_chunks, nested_chunk)) == () + + +def test_chunk_creator_skips_full_dump_of_model_response_stream_chunks( + logging_obj: Logging, +) -> None: + chunks: Final = ( + ModelResponseStream( + id="chunk-1", + created=1, + model="claude-model", + choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="hello"))], + ), + ModelResponseStream( + id="chunk-2", + created=1, + model="claude-model", + choices=[StreamingChoices(index=0, delta=Delta(content=" world"))], + custom_chunk_attribute="custom value", + ), + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="claude-model", + custom_llm_provider="anthropic", + logging_obj=logging_obj, + ) + model_dump: Final = BaseModel.model_dump.__code__ + + outputs, calls = _chunk_creator_calls(wrapper, chunks, frozenset({model_dump})) + emitted_chunks: Final = tuple(output for output in outputs if output is not None) + full_upstream_dumps: Final = tuple( + call + for call in calls + if any(call.arguments["self"] is chunk for chunk in chunks) and call.arguments["include"] is None + ) + + assert full_upstream_dumps == () + assert [chunk.choices[0].delta.content for chunk in emitted_chunks] == ["hello", " world"] + assert emitted_chunks[1].choices[0].delta.role is None + assert getattr(emitted_chunks[1], "custom_chunk_attribute") == "custom value" + + + +def test_openai_chunk_creator_keeps_multiple_choice_logprobs_and_extras( + logging_obj: Logging, +) -> None: + chunk: Final = ChatCompletionChunk( + id="chunk-with-multiple-choices", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + citations=["https://a"], + choices=[ + { + "index": 3, + "delta": {"content": "first"}, + "finish_reason": None, + "logprobs": {"content": []}, + }, + { + "index": 7, + "delta": {"content": "second"}, + "finish_reason": None, + "custom_choice_attribute": "choice extra", + }, + ], + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="gpt-4o", + custom_llm_provider="openai", + logging_obj=logging_obj, + ) + + result: Final = wrapper.chunk_creator(chunk=chunk) + + assert result is not None + assert [choice.index for choice in result.choices] == [3, 7] + result_logprobs = result.choices[0].logprobs + chunk_logprobs = chunk.choices[0].logprobs + assert result_logprobs is not None + assert chunk_logprobs is not None + assert result_logprobs.model_dump() == chunk_logprobs.model_dump() + assert getattr(result.choices[1], "custom_choice_attribute") == "choice extra" + assert getattr(result, "citations") == ["https://a"] + + +@pytest.mark.parametrize( + ("choice_extras", "expected_choice_keys", "expected_choice_constructions"), + [ + pytest.param( + { + "content_filter_results": {"hate": {"filtered": False, "severity": "safe"}}, + "message": {"role": "assistant", "content": "hello"}, + }, + ["content_filter_results", "index", "delta"], + 1, + id="azure-content-filter-results", + ), + pytest.param( + {"enhancements": {"grounding": {"lines": []}}}, + ["index", "delta", "enhancements"], + 2, + id="enhancements", + ), + ], +) +def test_chunk_creator_keeps_upstream_choice_extras_in_upstream_order( + logging_obj: Logging, + choice_extras: dict[str, object], + expected_choice_keys: list[str], + expected_choice_constructions: int, +) -> None: + chunk: Final = ChatCompletionChunk( + id="chunk-with-choice-extras", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 0, "delta": {"content": "hello"}, "finish_reason": None, **choice_extras}], + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="gpt-4o", + custom_llm_provider="azure", + logging_obj=logging_obj, + ) + streaming_choices_init: Final = StreamingChoices.__init__.__code__ + + outputs, calls = _chunk_creator_calls(wrapper, [chunk], frozenset({streaming_choices_init})) + + result: Final = outputs[0] + assert result is not None + choice_dump: Final = result.choices[0].model_dump(exclude_none=True) + assert list(choice_dump) == expected_choice_keys + assert choice_dump["delta"]["content"] == "hello" + assert {key: choice_dump[key] for key in choice_extras if key != "message"} == { + key: value for key, value in choice_extras.items() if key != "message" + } + assert len(calls) == expected_choice_constructions + + +def test_chunk_creator_keeps_upstream_index_on_single_choice_chunk(logging_obj: Logging) -> None: + chunk: Final = ChatCompletionChunk( + id="chunk-with-index-1", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 1, "delta": {"content": "hello"}, "finish_reason": None}], + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="gpt-4o", + custom_llm_provider="openai", + logging_obj=logging_obj, + ) + + result: Final = wrapper.chunk_creator(chunk=chunk) + + assert result is not None + assert result.choices[0].index == 1 + assert result.choices[0].delta.content == "hello" + + +def _later_chunk_outputs( + custom_llm_provider: str, later_chunks: Sequence[ChatCompletionChunk], logging_obj: Logging +) -> tuple[ModelResponseStream, ...]: + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="gpt-4o", + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) + wrapper.chunk_creator(chunk=_openai_chunk("chunk-0", {"role": "assistant", "content": ""})) + outputs: Final = tuple(wrapper.chunk_creator(chunk=chunk) for chunk in later_chunks) + return tuple(output for output in outputs if output is not None) + + +def _wire_dump(chunk: ModelResponseStream) -> dict[str, object]: + return chunk.model_dump(exclude_none=True, exclude_unset=True) + + +def test_openai_chunk_creator_keeps_single_choice_logprobs_on_later_chunks(logging_obj: Logging) -> None: + logprobs: Final = {"content": [{"token": "hello", "logprob": -0.25, "bytes": [104], "top_logprobs": []}]} + chunk: Final = ChatCompletionChunk( + id="chunk-1", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 0, "delta": {"content": "hello"}, "finish_reason": None, "logprobs": logprobs}], + ) + + outputs: Final = _later_chunk_outputs("openai", [chunk], logging_obj) + + assert [_wire_dump(output)["choices"][0]["logprobs"] for output in outputs] == [logprobs] + + +def test_azure_chunk_creator_streams_choice_content_filter_results_on_later_chunks(logging_obj: Logging) -> None: + filter_results: Final = {"hate": {"filtered": False, "severity": "safe"}} + chunk: Final = ChatCompletionChunk( + id="chunk-1", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[ + {"index": 0, "delta": {"content": "hello"}, "finish_reason": None, "content_filter_results": filter_results} + ], + ) + + outputs: Final = _later_chunk_outputs("azure", [chunk], logging_obj) + + assert [_wire_dump(output)["choices"][0]["content_filter_results"] for output in outputs] == [filter_results] + assert ["content_filter_results" in output.choices[0].model_fields_set for output in outputs] == [True] + + +@pytest.mark.parametrize( + "special_delta", + [ + pytest.param({"audio": {"id": "audio_1", "data": "AAAA", "transcript": "hi"}}, id="audio"), + pytest.param({"images": [{"type": "image_url", "image_url": {"url": "data:,"}, "index": 0}]}, id="images"), + ], +) +def test_openai_chunk_creator_dumps_special_delta_attributes_like_other_deltas( + logging_obj: Logging, special_delta: dict[str, object] +) -> None: + chunk: Final = ChatCompletionChunk.model_construct( + id="chunk-1", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[{"index": 0, "delta": special_delta, "finish_reason": None}], + ) + + outputs: Final = _later_chunk_outputs("openai", [chunk], logging_obj) + + assert [output.model_dump()["choices"][0]["delta"] for output in outputs] == [ + { + "audio": None, + "content": None, + "function_call": None, + "provider_specific_fields": None, + "role": None, + "tool_calls": None, + **special_delta, + } + ] + + +def test_openai_chunk_creator_keeps_custom_tool_name_key_on_later_chunks( + logging_obj: Logging, +) -> None: + chunks: Final = ( + ChatCompletionChunk.model_construct( + id="chunk-1", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[ + { + "index": 0, + "delta": { + "role": "assistant", + "tool_calls": [ + {"index": 0, "id": "call_1", "type": "custom", "custom": {"name": "run_python", "input": ""}} + ], + }, + "finish_reason": None, + } + ], + ), + ChatCompletionChunk.model_construct( + id="chunk-2", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[ + { + "index": 0, + "delta": {"tool_calls": [{"index": 0, "custom": {"input": "print"}}]}, + "finish_reason": None, + } + ], + ), + ) + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="gpt-4o", + custom_llm_provider="openai", + logging_obj=logging_obj, + ) + + emitted_chunks: Final = tuple( + output for output in (wrapper.chunk_creator(chunk=chunk) for chunk in chunks) if output is not None + ) + + assert [chunk.model_dump()["choices"][0]["delta"]["tool_calls"][0]["custom"] for chunk in emitted_chunks] == [ + {"name": "run_python", "input": ""}, + {"name": None, "input": "print"}, + ] + + def test_chunk_creator_preserves_hidden_provider_specific_fields_from_parsed_chunk(): wrapper = CustomStreamWrapper( completion_stream=None, @@ -5906,72 +7298,6 @@ def test_unit_test_perplexity_citations_chunk(): assert "citations" in response_chunk -def test_mock_response_iterator_tool_use(): - """ - Relevant Issue: https://github.com/BerriAI/litellm/issues/7364 - """ - from litellm.llms.bedrock.chat.invoke_handler import MockResponseIterator - from litellm.types.utils import ( - ChatCompletionMessageToolCall, - Choices, - CompletionTokensDetailsWrapper, - Function, - Message, - PromptTokensDetailsWrapper, - Usage, - ) - - litellm.set_verbose = False - response = ModelResponse( - id="chatcmpl-Ai8KRI5vJPZXQ9SQvEJfTVuVqkyEZ", - created=1735081811, - model="o1-2024-12-17", - object="chat.completion", - system_fingerprint="fp_e6d02d4a78", - choices=[ - Choices( - finish_reason="tool_calls", - index=0, - message=Message( - content=None, - role="assistant", - tool_calls=[ - ChatCompletionMessageToolCall( - function=Function( - arguments='{"location":"San Francisco, CA","unit":"fahrenheit"}', - name="get_current_weather", - ), - id="call_BfRX2S7YCKL0BtxbWMl89ZNk", - type="function", - ) - ], - function_call=None, - ), - ) - ], - usage=Usage( - completion_tokens=1955, - prompt_tokens=85, - total_tokens=2040, - completion_tokens_details=CompletionTokensDetailsWrapper( - accepted_prediction_tokens=0, - audio_tokens=0, - reasoning_tokens=1920, - rejected_prediction_tokens=0, - text_tokens=None, - ), - prompt_tokens_details=PromptTokensDetailsWrapper( - audio_tokens=0, cached_tokens=0, text_tokens=None, image_tokens=None - ), - ), - service_tier=None, - ) - completion_stream = MockResponseIterator(model_response=response) - response_chunk = completion_stream._chunk_parser(chunk_data=response) - - assert response_chunk["tool_use"] is not None - - def test_is_delta_empty(): from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.types.utils import Delta @@ -6074,6 +7400,59 @@ class TestStableStreamingResponseId: assert wrapper.model_response_creator().id == "chatcmpl-from-provider" +class TestContinuationDisqualifiers: + """The mid-stream continuation eligibility hinges on classifying which deltas + carry output a text-only prefill cannot represent.""" + + @pytest.mark.parametrize( + "field", + [ + "tool_calls", + "function_call", + "reasoning_content", + "thinking_blocks", + "reasoning_items", + "audio", + "images", + "annotations", + ], + ) + def test_disqualifying_fields_flagged(self, field): + # reasoning_content included: it reaches the caller as visible reasoning a + # text-only prefill cannot carry, so a reasoning fallback would re-derive it + assert CustomStreamWrapper._delta_disqualifies_continuation({field: [{"x": 1}]}) is True + + @pytest.mark.parametrize("delta", [{"content": "hi"}, {}, {"role": "assistant"}]) + def test_plain_text_not_flagged(self, delta): + assert CustomStreamWrapper._delta_disqualifies_continuation(delta) is False + + def test_non_mapping_delta_is_safe(self): + assert CustomStreamWrapper._delta_disqualifies_continuation(object()) is False + + def test_accumulate_grows_text_and_leaves_flag_clear_for_plain_text(self): + wrapper = object.__new__(CustomStreamWrapper) + wrapper.response_uptil_now = "" + wrapper._emitted_disqualifying_content = False + + wrapper._accumulate_streamed_delta({"content": "Hel"}) + wrapper._accumulate_streamed_delta({"content": "lo"}) + + assert wrapper.response_uptil_now == "Hello" + assert wrapper._emitted_disqualifying_content is False + + def test_accumulate_latches_flag_on_disqualifying_delta(self): + wrapper = object.__new__(CustomStreamWrapper) + wrapper.response_uptil_now = "" + wrapper._emitted_disqualifying_content = False + + wrapper._accumulate_streamed_delta({"content": "Hi"}) + wrapper._accumulate_streamed_delta({"tool_calls": [{"index": 0}]}) + wrapper._accumulate_streamed_delta({"content": "there"}) + + assert wrapper.response_uptil_now == "Hithere" + assert wrapper._emitted_disqualifying_content is True + + @pytest.mark.asyncio async def test_async_stream_without_usage_counts_tokens_off_the_event_loop(): from tests.large_text import text @@ -6419,6 +7798,54 @@ def validate_last_format(chunk): ), "'finish_reason' should be a string." +class _RecordingProviderStream: + def __init__(self) -> None: + self.close_calls = 0 + + def __iter__(self) -> "_RecordingProviderStream": + return self + + def __next__(self) -> object: + raise StopIteration + + def close(self) -> None: + self.close_calls += 1 + + +def _wrapper_over(stream: _RecordingProviderStream) -> CustomStreamWrapper: + return CustomStreamWrapper( + completion_stream=stream, + model="gpt-5.6", + logging_obj=MagicMock(), + custom_llm_provider="openai", + ) + + +def test_close_releases_the_provider_stream_once_and_ends_iteration(): + stream: Final = _RecordingProviderStream() + wrapper: Final = _wrapper_over(stream) + wrapper.close() + wrapper.close() + assert stream.close_calls == 1 + with pytest.raises(StopIteration): + next(wrapper) + + +def test_leaving_a_with_block_closes_the_provider_stream(): + stream: Final = _RecordingProviderStream() + with _wrapper_over(stream): + pass + assert stream.close_calls == 1 + + +@pytest.mark.asyncio +async def test_leaving_an_async_with_block_closes_the_provider_stream(): + stream: Final = _RecordingProviderStream() + async with _wrapper_over(stream): + pass + assert stream.close_calls == 1 + + def _chunk_with_list_content(text: str, chunk_id: str, created: int) -> ModelResponseStream: return ModelResponseStream( id=chunk_id,