mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(completions): drop Azure's empty prompt-filter frame when the proxy asked for stream usage
This commit is contained in:
parent
f9ee31f2b2
commit
cc6b89bb64
5 changed files with 46 additions and 26 deletions
|
|
@ -19,7 +19,9 @@ def _all_values_none(values: Iterable[object]) -> bool:
|
|||
return all(value is None for value in values)
|
||||
|
||||
|
||||
def is_usage_only_chunk(chunk: ModelResponseStream) -> bool:
|
||||
if getattr(chunk, "usage", None) is None:
|
||||
def is_injected_stream_usage_artifact(chunk: object) -> bool:
|
||||
if not isinstance(chunk, ModelResponseStream):
|
||||
return False
|
||||
if chunk.provider_specific_fields is not None:
|
||||
return False
|
||||
return all(is_empty_streaming_choice(choice) for choice in chunk.choices or [])
|
||||
|
|
|
|||
|
|
@ -376,7 +376,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import (
|
|||
SensitiveDataMasker,
|
||||
mask_sensitive_keys,
|
||||
)
|
||||
from litellm.litellm_core_utils.stream_usage_chunks import is_empty_streaming_choice
|
||||
from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import (
|
||||
deployment_drops_params,
|
||||
drop_non_string_tool_result_texts,
|
||||
|
|
@ -10148,14 +10148,6 @@ def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes:
|
|||
return chunk.model_dump_json(exclude_none=True, exclude_unset=True)
|
||||
|
||||
|
||||
def _is_injected_stream_usage_artifact(chunk: object) -> bool:
|
||||
if not isinstance(chunk, ModelResponseStream):
|
||||
return False
|
||||
if chunk.provider_specific_fields is not None:
|
||||
return False
|
||||
return all(is_empty_streaming_choice(choice) for choice in chunk.choices or [])
|
||||
|
||||
|
||||
async def _apply_streaming_chunk_hooks(
|
||||
*,
|
||||
chunk: Any,
|
||||
|
|
@ -10535,7 +10527,7 @@ async def async_data_generator(
|
|||
fallback_model_from_metadata=fallback_model_from_metadata,
|
||||
)
|
||||
|
||||
if strip_stream_usage and _is_injected_stream_usage_artifact(chunk):
|
||||
if strip_stream_usage and is_injected_stream_usage_artifact(chunk):
|
||||
if pending_fallback_event:
|
||||
yield _format_fallback_metadata_sse_event(
|
||||
fallback_model=fallback_model_from_metadata,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.stream_usage_chunks import is_usage_only_chunk
|
||||
from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact
|
||||
from litellm.types.utils import Choices, ModelResponse, ModelResponseStream, TextChoices, TextCompletionResponse
|
||||
|
||||
|
||||
|
|
@ -60,8 +60,8 @@ class TextCompletionStreamWrapper:
|
|||
except Exception as e:
|
||||
raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}")
|
||||
|
||||
def _is_injected_usage_chunk(self, chunk: ModelResponseStream) -> bool:
|
||||
return self.usage_injected_by_proxy and is_usage_only_chunk(chunk)
|
||||
def _is_injected_usage_artifact(self, chunk: ModelResponseStream) -> bool:
|
||||
return self.usage_injected_by_proxy and is_injected_stream_usage_artifact(chunk)
|
||||
|
||||
def __next__(self):
|
||||
# model_response = ModelResponse(stream=True, model=self.model)
|
||||
|
|
@ -70,7 +70,7 @@ class TextCompletionStreamWrapper:
|
|||
for chunk in self.completion_stream:
|
||||
if chunk == "None" or chunk is None:
|
||||
raise Exception
|
||||
if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_chunk(chunk):
|
||||
if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_artifact(chunk):
|
||||
continue
|
||||
return self.convert_to_text_completion_object(chunk=chunk)
|
||||
raise StopIteration
|
||||
|
|
@ -92,7 +92,7 @@ class TextCompletionStreamWrapper:
|
|||
async for chunk in self.completion_stream:
|
||||
if chunk == "None" or chunk is None:
|
||||
raise Exception
|
||||
if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_chunk(chunk):
|
||||
if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_artifact(chunk):
|
||||
continue
|
||||
return self.convert_to_text_completion_object(chunk=chunk)
|
||||
raise StopIteration
|
||||
|
|
|
|||
|
|
@ -13581,29 +13581,29 @@ def _stream_usage_generator_chunks():
|
|||
|
||||
|
||||
def test_is_injected_stream_usage_artifact():
|
||||
from litellm.proxy.proxy_server import _is_injected_stream_usage_artifact
|
||||
from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact
|
||||
from litellm.types.utils import ModelResponseStream, Usage
|
||||
|
||||
content_chunk, finish_chunk, empty_choices_usage_chunk = _stream_usage_test_chunks()
|
||||
assert _is_injected_stream_usage_artifact(empty_choices_usage_chunk) is True
|
||||
assert is_injected_stream_usage_artifact(empty_choices_usage_chunk) is True
|
||||
|
||||
synthetic_final_chunk = ModelResponseStream(model="gpt-5.4-nano")
|
||||
synthetic_final_chunk.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238)
|
||||
assert _is_injected_stream_usage_artifact(synthetic_final_chunk) is True
|
||||
assert is_injected_stream_usage_artifact(synthetic_final_chunk) is True
|
||||
|
||||
azure_prompt_filter_chunk = ModelResponseStream(model="gpt-5.4-nano", choices=[])
|
||||
assert _is_injected_stream_usage_artifact(azure_prompt_filter_chunk) is True
|
||||
assert is_injected_stream_usage_artifact(azure_prompt_filter_chunk) is True
|
||||
|
||||
assert _is_injected_stream_usage_artifact(content_chunk) is False
|
||||
assert _is_injected_stream_usage_artifact(finish_chunk) is False
|
||||
assert is_injected_stream_usage_artifact(content_chunk) is False
|
||||
assert is_injected_stream_usage_artifact(finish_chunk) is False
|
||||
|
||||
content_chunk_with_usage, finish_chunk_with_usage, _ = _stream_usage_test_chunks()
|
||||
content_chunk_with_usage.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238)
|
||||
finish_chunk_with_usage.usage = Usage(prompt_tokens=50, completion_tokens=188, total_tokens=238)
|
||||
assert _is_injected_stream_usage_artifact(content_chunk_with_usage) is False
|
||||
assert _is_injected_stream_usage_artifact(finish_chunk_with_usage) is False
|
||||
assert is_injected_stream_usage_artifact(content_chunk_with_usage) is False
|
||||
assert is_injected_stream_usage_artifact(finish_chunk_with_usage) is False
|
||||
|
||||
assert _is_injected_stream_usage_artifact({"usage": {"prompt_tokens": 1}}) is False
|
||||
assert is_injected_stream_usage_artifact({"usage": {"prompt_tokens": 1}}) is False
|
||||
|
||||
|
||||
async def _collect_async_data_generator_frames(request_data: dict) -> list:
|
||||
|
|
|
|||
|
|
@ -203,3 +203,29 @@ def test_text_completion_stream_keeps_a_finish_frame_but_hides_the_usage_the_pro
|
|||
frames: Final = list(wrapper)
|
||||
|
||||
assert [(frame.choices[0].finish_reason, frame.usage) for frame in frames] == [("stop", None)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("async_stream", (False, True))
|
||||
async def test_text_completion_stream_hides_the_empty_prompt_filter_frame_when_the_proxy_asked_for_usage(
|
||||
async_stream: bool,
|
||||
) -> None:
|
||||
prompt_filter_frame: Final = ModelResponseStream(id="", choices=[])
|
||||
upstream: Final = (prompt_filter_frame, *_stream_with_trailing_usage_chunk())
|
||||
|
||||
async def chunks() -> AsyncIterator[ModelResponseStream]:
|
||||
for chunk in upstream:
|
||||
yield chunk
|
||||
|
||||
wrapper: Final = TextCompletionStreamWrapper(
|
||||
completion_stream=chunks() if async_stream else iter(upstream),
|
||||
model="gpt-5.4-mini",
|
||||
stream_options={"include_usage": True},
|
||||
usage_injected_by_proxy=True,
|
||||
)
|
||||
frames: Final = [frame async for frame in wrapper] if async_stream else list(wrapper)
|
||||
|
||||
assert [(frame.choices[0].text, frame.choices[0].finish_reason) for frame in frames] == [
|
||||
("Hi", None),
|
||||
(None, "length"),
|
||||
]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue