fix(completions): drop Azure's empty prompt-filter frame when the proxy asked for stream usage

This commit is contained in:
mateo-berri 2026-10-28 23:05:07 -07:00
parent f9ee31f2b2
commit cc6b89bb64
5 changed files with 46 additions and 26 deletions

View file

@ -19,7 +19,9 @@ def _all_values_none(values: Iterable[object]) -> bool:
return all(value is None for value in values)
def is_usage_only_chunk(chunk: ModelResponseStream) -> bool:
if getattr(chunk, "usage", None) is None:
def is_injected_stream_usage_artifact(chunk: object) -> bool:
if not isinstance(chunk, ModelResponseStream):
return False
if chunk.provider_specific_fields is not None:
return False
return all(is_empty_streaming_choice(choice) for choice in chunk.choices or [])

View file

@ -376,7 +376,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import (
SensitiveDataMasker,
mask_sensitive_keys,
)
from litellm.litellm_core_utils.stream_usage_chunks import is_empty_streaming_choice
from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact
from litellm.llms.anthropic.messages.convertible_blocks import (
deployment_drops_params,
drop_non_string_tool_result_texts,
@ -10148,14 +10148,6 @@ def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes:
return chunk.model_dump_json(exclude_none=True, exclude_unset=True)
def _is_injected_stream_usage_artifact(chunk: object) -> bool:
if not isinstance(chunk, ModelResponseStream):
return False
if chunk.provider_specific_fields is not None:
return False
return all(is_empty_streaming_choice(choice) for choice in chunk.choices or [])
async def _apply_streaming_chunk_hooks(
*,
chunk: Any,
@ -10535,7 +10527,7 @@ async def async_data_generator(
fallback_model_from_metadata=fallback_model_from_metadata,
)
if strip_stream_usage and _is_injected_stream_usage_artifact(chunk):
if strip_stream_usage and is_injected_stream_usage_artifact(chunk):
if pending_fallback_event:
yield _format_fallback_metadata_sse_event(
fallback_model=fallback_model_from_metadata,

View file

@ -1,6 +1,6 @@
from typing import Final
from litellm.litellm_core_utils.stream_usage_chunks import is_usage_only_chunk
from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact
from litellm.types.utils import Choices, ModelResponse, ModelResponseStream, TextChoices, TextCompletionResponse
@ -60,8 +60,8 @@ class TextCompletionStreamWrapper:
except Exception as e:
raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}")
def _is_injected_usage_chunk(self, chunk: ModelResponseStream) -> bool:
return self.usage_injected_by_proxy and is_usage_only_chunk(chunk)
def _is_injected_usage_artifact(self, chunk: ModelResponseStream) -> bool:
return self.usage_injected_by_proxy and is_injected_stream_usage_artifact(chunk)
def __next__(self):
# model_response = ModelResponse(stream=True, model=self.model)
@ -70,7 +70,7 @@ class TextCompletionStreamWrapper:
for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_chunk(chunk):
if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_artifact(chunk):
continue
return self.convert_to_text_completion_object(chunk=chunk)
raise StopIteration
@ -92,7 +92,7 @@ class TextCompletionStreamWrapper:
async for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_chunk(chunk):
if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_artifact(chunk):
continue
return self.convert_to_text_completion_object(chunk=chunk)
raise StopIteration

View file

@ -13581,29 +13581,29 @@ def _stream_usage_generator_chunks():
def test_is_injected_stream_usage_artifact():
from litellm.proxy.proxy_server import _is_injected_stream_usage_artifact
from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact
from litellm.types.utils import ModelResponseStream, Usage
content_chunk, finish_chunk, empty_choices_usage_chunk = _stream_usage_test_chunks()
assert _is_injected_stream_usage_artifact(empty_choices_usage_chunk) is True
assert is_injected_stream_usage_artifact(empty_choices_usage_chunk) is True
synthetic_final_chunk = ModelResponseStream(model="gpt-5.4-nano")
synthetic_final_chunk.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238)
assert _is_injected_stream_usage_artifact(synthetic_final_chunk) is True
assert is_injected_stream_usage_artifact(synthetic_final_chunk) is True
azure_prompt_filter_chunk = ModelResponseStream(model="gpt-5.4-nano", choices=[])
assert _is_injected_stream_usage_artifact(azure_prompt_filter_chunk) is True
assert is_injected_stream_usage_artifact(azure_prompt_filter_chunk) is True
assert _is_injected_stream_usage_artifact(content_chunk) is False
assert _is_injected_stream_usage_artifact(finish_chunk) is False
assert is_injected_stream_usage_artifact(content_chunk) is False
assert is_injected_stream_usage_artifact(finish_chunk) is False
content_chunk_with_usage, finish_chunk_with_usage, _ = _stream_usage_test_chunks()
content_chunk_with_usage.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238)
finish_chunk_with_usage.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238)
assert _is_injected_stream_usage_artifact(content_chunk_with_usage) is False
assert _is_injected_stream_usage_artifact(finish_chunk_with_usage) is False
assert is_injected_stream_usage_artifact(content_chunk_with_usage) is False
assert is_injected_stream_usage_artifact(finish_chunk_with_usage) is False
assert _is_injected_stream_usage_artifact({"usage": {"prompt_tokens": 1}}) is False
assert is_injected_stream_usage_artifact({"usage": {"prompt_tokens": 1}}) is False
async def _collect_async_data_generator_frames(request_data: dict) -> list:

View file

@ -203,3 +203,29 @@ def test_text_completion_stream_keeps_a_finish_frame_but_hides_the_usage_the_pro
frames: Final = list(wrapper)
assert [(frame.choices[0].finish_reason, frame.usage) for frame in frames] == [("stop", None)]
@pytest.mark.asyncio
@pytest.mark.parametrize("async_stream", (False, True))
async def test_text_completion_stream_hides_the_empty_prompt_filter_frame_when_the_proxy_asked_for_usage(
async_stream: bool,
) -> None:
prompt_filter_frame: Final = ModelResponseStream(id="", choices=[])
upstream: Final = (prompt_filter_frame, *_stream_with_trailing_usage_chunk())
async def chunks() -> AsyncIterator[ModelResponseStream]:
for chunk in upstream:
yield chunk
wrapper: Final = TextCompletionStreamWrapper(
completion_stream=chunks() if async_stream else iter(upstream),
model="gpt-5.4-mini",
stream_options={"include_usage": True},
usage_injected_by_proxy=True,
)
frames: Final = [frame async for frame in wrapper] if async_stream else list(wrapper)
assert [(frame.choices[0].text, frame.choices[0].finish_reason) for frame in frames] == [
("Hi", None),
(None, "length"),
]