diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index e477044d95b..43a9f56b933 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -8,19 +8,17 @@ import httpx import pytest import respx + import litellm from litellm import ChatCompletionUsageBlock, stream_chunk_builder -from litellm.litellm_core_utils.streaming_chunk_builder_utils import ( - ChunkProcessor, - stream_chunk_builder_text_completion, -) +from litellm.types.utils import GenericStreamingChunk +from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor, stream_chunk_builder_text_completion from litellm.llms.anthropic.chat.handler import ModelResponseIterator from litellm.types.utils import ( ChatCompletionDeltaToolCall, ChatCompletionMessageToolCall, Delta, Function, - GenericStreamingChunk, ModelResponseStream, PromptTokensDetails, ServerToolUse, @@ -198,7 +196,11 @@ def test_get_combined_thinking_content_preserves_interleaved_blocks(): } ] ), - make_chunk(thinking_blocks=[{"type": "thinking", "thinking": None, "signature": "sig_block1"}]), + make_chunk( + thinking_blocks=[ + {"type": "thinking", "thinking": None, "signature": "sig_block1"} + ] + ), make_chunk( thinking_blocks=[ { @@ -216,10 +218,16 @@ def test_get_combined_thinking_content_preserves_interleaved_blocks(): } ] ), - make_chunk(thinking_blocks=[{"type": "thinking", "thinking": None, "signature": "sig_block2"}]), + make_chunk( + thinking_blocks=[ + {"type": "thinking", "thinking": None, "signature": "sig_block2"} + ] + ), ] - thinking_chunks = [chunk for chunk in chunks if chunk["choices"][0]["delta"].get("thinking_blocks")] + thinking_chunks = [ + chunk for chunk in chunks if chunk["choices"][0]["delta"].get("thinking_blocks") + ] processor = ChunkProcessor(chunks=chunks) result = processor.get_combined_thinking_content(thinking_chunks) @@ -289,7 +297,9 @@ def test_cache_read_input_tokens_retained(): prompt_tokens=11779, total_tokens=11784, completion_tokens_details=None, - prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=11775), + prompt_tokens_details=PromptTokensDetails( + audio_tokens=None, cached_tokens=11775 + ), cache_creation_input_tokens=4, cache_read_input_tokens=11775, ), @@ -323,7 +333,9 @@ def test_cache_read_input_tokens_retained(): prompt_tokens=0, total_tokens=214, completion_tokens_details=None, - prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=0), + prompt_tokens_details=PromptTokensDetails( + audio_tokens=None, cached_tokens=0 + ), cache_creation_input_tokens=0, cache_read_input_tokens=0, ), @@ -383,7 +395,10 @@ def test_streaming_preserves_anthropic_1hr_cache_creation_breakdown(): ) # Sanity: the delta event genuinely lacks the breakdown - this is the input # condition that used to defeat cost calc. - assert getattr(message_delta_usage.prompt_tokens_details, "cache_creation_token_details", None) is None + assert ( + getattr(message_delta_usage.prompt_tokens_details, "cache_creation_token_details", None) + is None + ) def _usage_chunk(usage, finish_reason): return ModelResponseStream( @@ -502,7 +517,9 @@ def test_cache_read_input_tokens_retained_genericstreamingchunk(): prompt_tokens=1234, total_tokens=1239, completion_tokens_details=None, - prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=543).model_dump(), + prompt_tokens_details=PromptTokensDetails( + audio_tokens=None, cached_tokens=543 + ).model_dump(), ), index=2, ) @@ -519,7 +536,6 @@ def test_cache_read_input_tokens_retained_genericstreamingchunk(): assert usage.prompt_tokens_details.cached_tokens == 543 - def test_stream_chunk_builder_litellm_usage_chunks(): """ Validate ChunkProcessor.calculate_usage uses provided usage fields from streaming chunks @@ -593,7 +609,9 @@ def test_stream_chunk_builder_litellm_usage_chunks(): chunks = [chunk1, chunk2] processor = ChunkProcessor(chunks=chunks) - usage = processor.calculate_usage(chunks=chunks, model="gemini/gemini-2.5-flash-lite", completion_output="") + usage = processor.calculate_usage( + chunks=chunks, model="gemini/gemini-2.5-flash-lite", completion_output="" + ) assert usage.prompt_tokens == 50 assert usage.completion_tokens == 27 @@ -637,11 +655,15 @@ def test_calculate_usage_honors_openai_sdk_completion_usage_chunks(): provider_specific_fields=None, stream_options={"include_usage": True}, ) - usage_chunk.usage = CompletionUsage(prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704) + usage_chunk.usage = CompletionUsage( + prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704 + ) assert type(usage_chunk.usage) is CompletionUsage chunks = [content_chunk, usage_chunk] - usage = ChunkProcessor(chunks=chunks).calculate_usage(chunks=chunks, model="mantle-claude", completion_output="") + usage = ChunkProcessor(chunks=chunks).calculate_usage( + chunks=chunks, model="mantle-claude", completion_output="" + ) assert usage.prompt_tokens == 20 assert usage.completion_tokens == 60 @@ -664,7 +686,9 @@ def test_get_model_from_chunks_azure_model_router(): {"model": "gpt-4.1-nano-2025-04-14", "id": "chatcmpl-123", "choices": []}, ] - result = ChunkProcessor._get_model_from_chunks(chunks=chunks, first_chunk_model="azure-model-router") + result = ChunkProcessor._get_model_from_chunks( + chunks=chunks, first_chunk_model="azure-model-router" + ) # Should return the actual model, not the request model assert result == "gpt-4.1-nano-2025-04-14" @@ -675,7 +699,9 @@ def test_get_model_from_chunks_azure_model_router(): {"model": "gpt-4", "id": "chatcmpl-456", "choices": []}, ] - result_same = ChunkProcessor._get_model_from_chunks(chunks=chunks_same_model, first_chunk_model="gpt-4") + result_same = ChunkProcessor._get_model_from_chunks( + chunks=chunks_same_model, first_chunk_model="gpt-4" + ) # Should return the first chunk's model when all are the same assert result_same == "gpt-4" @@ -751,7 +777,9 @@ def test_stream_chunk_builder_anthropic_web_search(): chunks = [chunk1, chunk2] processor = ChunkProcessor(chunks=chunks) - usage = processor.calculate_usage(chunks=chunks, model="claude-sonnet-4-5-20250929", completion_output="") + usage = processor.calculate_usage( + chunks=chunks, model="claude-sonnet-4-5-20250929", completion_output="" + ) assert usage.prompt_tokens == 50 assert usage.completion_tokens == 27 @@ -903,11 +931,15 @@ def test_stream_chunk_builder_dict_snapshot_preserves_hidden_provider_fields(): ], ) chunk_dict = chunk.model_dump() - chunk_dict["_hidden_params"] = {"provider_specific_fields": {"traffic_type": "default"}} + chunk_dict["_hidden_params"] = { + "provider_specific_fields": {"traffic_type": "default"} + } response = stream_chunk_builder(chunks=[chunk_dict]) assert response is not None - assert response._hidden_params["provider_specific_fields"]["traffic_type"] == "default" + assert ( + response._hidden_params["provider_specific_fields"]["traffic_type"] == "default" + ) def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_chunks(): @@ -952,7 +984,10 @@ def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_chunks(): assert getattr(response, "vertex_ai_grounding_metadata") == grounding_metadata assert getattr(response, "vertex_ai_url_context_metadata") == url_context_metadata assert response._hidden_params["vertex_ai_grounding_metadata"] == grounding_metadata - assert response._hidden_params["vertex_ai_url_context_metadata"] == url_context_metadata + assert ( + response._hidden_params["vertex_ai_url_context_metadata"] + == url_context_metadata + ) dumped = response.model_dump() assert dumped["vertex_ai_grounding_metadata"] == grounding_metadata @@ -999,7 +1034,9 @@ def test_stream_chunk_builder_uses_assembled_model_for_provider_metadata(): def test_stream_chunk_builder_propagates_vertex_ai_safety_results(): """Assembled response must expose safety data under the non-streaming field name.""" - safety_ratings = [[{"category": "HARM_CATEGORY_HATE_SPEECH", "probability": "NEGLIGIBLE"}]] + safety_ratings = [ + [{"category": "HARM_CATEGORY_HATE_SPEECH", "probability": "NEGLIGIBLE"}] + ] chunk = ModelResponseStream( id="chatcmpl-vertex-safety", @@ -1041,12 +1078,18 @@ def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_dict_chunks(): ) ], ).model_dump() - chunk_dict["_hidden_params"] = {"vertex_ai_grounding_metadata": [{"webSearchQueries": ["test query"]}]} + chunk_dict["_hidden_params"] = { + "vertex_ai_grounding_metadata": [{"webSearchQueries": ["test query"]}] + } response = stream_chunk_builder(chunks=[chunk_dict]) assert response is not None - assert getattr(response, "vertex_ai_grounding_metadata") == [{"webSearchQueries": ["test query"]}] - assert response.model_dump()["vertex_ai_grounding_metadata"] == [{"webSearchQueries": ["test query"]}] + assert getattr(response, "vertex_ai_grounding_metadata") == [ + {"webSearchQueries": ["test query"]} + ] + assert response.model_dump()["vertex_ai_grounding_metadata"] == [ + {"webSearchQueries": ["test query"]} + ] def test_cost_field_in_usage_chunks(): @@ -1055,21 +1098,29 @@ def test_cost_field_in_usage_chunks(): id="chatcmpl-1", created=1745513206, model="openrouter/claude", - choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi"))], + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) + ], usage=chunk1_usage, ) - chunk2_usage = Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.00025) + chunk2_usage = Usage( + completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.00025 + ) chunk2 = ModelResponseStream( id="chatcmpl-1", created=1745513207, model="openrouter/claude", - choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))], + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], usage=chunk2_usage, ) processor = ChunkProcessor(chunks=[chunk1, chunk2]) - usage = processor.calculate_usage(chunks=[chunk1, chunk2], model="openrouter/claude", completion_output="Hi") + usage = processor.calculate_usage( + chunks=[chunk1, chunk2], model="openrouter/claude", completion_output="Hi" + ) assert hasattr(usage, "cost") assert usage.cost == 0.00025 @@ -1152,19 +1203,25 @@ def test_prompt_tokens_details_survive_later_usage_chunk_without_details(): id="chatcmpl-1", created=1745513206, model="openai/gpt-5.6-sol", - choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi"))], + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) + ], usage=Usage( prompt_tokens=6017, completion_tokens=4, total_tokens=6021, - prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=6004, cache_write_tokens=10), + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=6004, cache_write_tokens=10 + ), ), ) chunk_without_details = ModelResponseStream( id="chatcmpl-1", created=1745513207, model="openai/gpt-5.6-sol", - choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))], + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], usage=Usage(prompt_tokens=6017, completion_tokens=4, total_tokens=6021), ) @@ -1502,7 +1559,9 @@ def test_calculate_usage_fills_unknown_split_from_reasoning_estimate( assert usage.completion_tokens_details.text_tokens == expected_text_tokens -def _openai_chunk(choices: Sequence[Mapping[str, object]], usage: Mapping[str, int] | None = None) -> dict[str, object]: +def _openai_chunk( + choices: Sequence[Mapping[str, object]], usage: Mapping[str, int] | None = None +) -> dict[str, object]: base: Final = { "id": "chatcmpl-lit6552", "object": "chat.completion.chunk", @@ -1937,11 +1996,7 @@ def test_stream_chunk_builder_litellm_tool_call(respx_mock: respx.MockRouter) -> ) assert [tool_call.to_dict() for tool_call in response.choices[0].message.tool_calls] == [ - { - "function": {"arguments": '{"city":"Boston"}', "name": "get_weather"}, - "id": "call_weather", - "type": "function", - } + {"function": {"arguments": '{"city":"Boston"}', "name": "get_weather"}, "id": "call_weather", "type": "function"} ] assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (5, 3, 8) @@ -1963,9 +2018,7 @@ def test_stream_chunk_builder_litellm_tool_call_regular_message(respx_mock: resp def test_stream_chunk_builder_multiple_tool_calls() -> None: - def tool_call_chunk( - *tool_calls: ChatCompletionDeltaToolCall, finish_reason: str | None = None - ) -> ModelResponseStream: + def tool_call_chunk(*tool_calls: ChatCompletionDeltaToolCall, finish_reason: str | None = None) -> ModelResponseStream: return _legacy_stream_chunk("openai/gpt-4o", tool_calls=tool_calls, finish_reason=finish_reason) chunks: Final = ( @@ -1976,9 +2029,7 @@ def test_stream_chunk_builder_multiple_tool_calls() -> None: ), tool_call_chunk(ChatCompletionDeltaToolCall(index=0, function=Function(arguments='{"base": 3, '))), tool_call_chunk( - ChatCompletionDeltaToolCall( - id="call_add", index=1, type="function", function=Function(name="add", arguments="") - ) + ChatCompletionDeltaToolCall(id="call_add", index=1, type="function", function=Function(name="add", arguments="")) ), tool_call_chunk(ChatCompletionDeltaToolCall(index=0, function=Function(arguments='"exponent": 5}'))), tool_call_chunk( @@ -1992,11 +2043,7 @@ def test_stream_chunk_builder_multiple_tool_calls() -> None: assert response is not None assert response.choices[0].finish_reason == "tool_calls" assert [tool_call.to_dict() for tool_call in response.choices[0].message.tool_calls] == [ - { - "function": {"arguments": '{"base": 3, "exponent": 5}', "name": "exponentiate"}, - "id": "call_exp", - "type": "function", - }, + {"function": {"arguments": '{"base": 3, "exponent": 5}', "name": "exponentiate"}, "id": "call_exp", "type": "function"}, {"function": {"arguments": '{"x": 1, "y": 2}', "name": "add"}, "id": "call_add", "type": "function"}, ] @@ -2030,7 +2077,9 @@ def test_grok_bug() -> None: response: Final = stream_chunk_builder(chunks=chunks) assert response is not None - assert json.loads(response.choices[0].message.tool_calls[0].function.arguments) == {"location": "Boston"} + assert json.loads(response.choices[0].message.tool_calls[0].function.arguments) == { + "location": "Boston" + } def test_stream_chunk_builder_openai_audio_output_usage() -> None: @@ -2147,9 +2196,7 @@ def test_stream_chunk_builder_keeps_usage_dict_set_on_a_typed_text_completion_ch @pytest.mark.parametrize("build", [litellm.stream_chunk_builder, litellm.stream_chunk_builder_text_completion]) -def test_stream_chunk_builder_rebuilds_dict_text_completion_chunks( - build: Callable[..., TextCompletionResponse], -) -> None: +def test_stream_chunk_builder_rebuilds_dict_text_completion_chunks(build: Callable[..., TextCompletionResponse]) -> None: def chunk(choices: list[dict[str, object]], **extra: object) -> dict[str, object]: return { "id": "cmpl-1", @@ -2286,22 +2333,6 @@ def test_stream_chunk_builder_rebuilds_each_text_completion_choice_by_index() -> assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (1, 4) -def test_stream_chunk_builder_keeps_null_text_completion_logprobs_fields_null() -> None: - response: Final = litellm.stream_chunk_builder( - chunks=[ - _text_completion_chunk(_text_choice(0, "Hello", 5, None)), - _text_completion_chunk(_text_choice(0, " world", 10, None, "length")), - ] - ) - - assert response.choices[0].logprobs.model_dump() == { - "tokens": ["Hello", " world"], - "token_logprobs": [-0.5, -0.5], - "top_logprobs": None, - "text_offset": [5, 10], - } - - def test_stream_chunk_builder_joins_content_part_list_delta(): """A provider streaming delta.content as a content-part list must aggregate as text.""" chunks = [ @@ -2367,6 +2398,22 @@ def test_delta_content_as_str_reads_text_from_non_mapping_parts(): assert delta_content_as_str([_TextPart()]) == "from attribute" +def test_stream_chunk_builder_keeps_null_text_completion_logprobs_fields_null() -> None: + response: Final = litellm.stream_chunk_builder( + chunks=[ + _text_completion_chunk(_text_choice(0, "Hello", 5, None)), + _text_completion_chunk(_text_choice(0, " world", 10, None, "length")), + ] + ) + + assert response.choices[0].logprobs.model_dump() == { + "tokens": ["Hello", " world"], + "token_logprobs": [-0.5, -0.5], + "top_logprobs": None, + "text_offset": [5, 10], + } + + def test_stream_chunk_builder_keeps_a_text_completion_stream_that_ends_on_a_chat_chunk() -> None: response: Final = litellm.stream_chunk_builder( chunks=[ @@ -2468,9 +2515,7 @@ def test_stream_chunk_builder_rebuilds_text_completion_choice_indexes_that_are_n ] ) - assert ( - tuple(choice.model_dump(include={"index", "text", "finish_reason"}) for choice in response.choices) == expected - ) + assert tuple(choice.model_dump(include={"index", "text", "finish_reason"}) for choice in response.choices) == expected def test_stream_chunk_builder_text_completion_survives_uncountable_prompt() -> None: @@ -2537,50 +2582,3 @@ def test_stream_chunk_builder_keeps_reasoning_items_on_text_only_replies(): assert response is not None assert response.choices[0].message.content == "Hello world" assert response.choices[0].message.reasoning_items == reasoning_items - - -def test_stream_chunk_builder_carries_the_bedrock_guardrail_trace_and_records_it_on_the_logging_object( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """A Bedrock stream reports its guardrail trace on the last chunk only; the built response carries it like the - non-streaming response and the call bills it as guardrail cost.""" - from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices - - monkeypatch.setattr( - litellm, - "model_cost", - { - "bedrock/guardrails": { - "litellm_provider": "bedrock", - "mode": "guardrails", - "guardrail_cost_per_unit": {"topicPolicyUnits": 0.00015}, - } - }, - ) - trace = {"guardrail": {"inputAssessment": {"gid": {"invocationMetrics": {"usage": {"topicPolicyUnits": 2}}}}}} - chunks = [ - ModelResponseStream(id="s", model="m", choices=[StreamingChoices(index=0, delta=Delta(content="hi"))]), - ModelResponseStream( - id="s", - model="m", - choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")], - provider_specific_fields={"trace": trace}, - ), - ] - logging_obj = Logging( - model="bedrock/m", - messages=[{"role": "user", "content": "q"}], - stream=True, - call_type="acompletion", - start_time=None, - litellm_call_id="call", - function_id="fn", - ) - logging_obj.model_call_details["litellm_params"] = {"metadata": {}} - built = stream_chunk_builder(chunks=chunks, messages=[{"role": "user", "content": "q"}], logging_obj=logging_obj) - assert built is not None - assert getattr(built, "trace") == trace - (entry,) = logging_obj.model_call_details["litellm_params"]["metadata"]["standard_logging_guardrail_information"] - assert entry["guardrail_usage"] == {"topicPolicyUnits": 2} - assert entry["guardrail_cost"] == pytest.approx(2 * 0.00015)