test(streaming): drive refusal-only deltas through the real stream parsers

This commit is contained in:
mateo-berri 2026-10-31 02:49:47 -07:00
parent 769ed7a145
commit c690783b04

View file

@ -1,9 +1,14 @@
import asyncio
import itertools
import json
import time
from typing import Final, NoReturn, Optional
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import httpx
from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
from openai.types.chat import ChatCompletionChunk
import pytest
import litellm
@ -16,6 +21,7 @@ from litellm.litellm_core_utils.streaming_handler import (
_ProviderChunkParsed,
)
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
Delta,
@ -157,31 +163,6 @@ def test_is_chunk_non_empty(initialized_custom_stream_wrapper: CustomStreamWrapp
)
def test_is_chunk_non_empty_with_refusal(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
"""Unit test if non-empty when the delta carries only a refusal"""
chunk: Final = {
"id": "chatcmpl-abc",
"object": "chat.completion.chunk",
"created": 1700000000,
"model": "m",
"choices": [
{
"index": 0,
"delta": {"refusal": "I can't help"},
"logprobs": None,
"finish_reason": None,
}
],
}
assert initialized_custom_stream_wrapper.is_chunk_non_empty(
completion_obj=MagicMock(),
model_response=ModelResponseStream(**chunk),
response_obj=MagicMock(),
)
def test_is_chunk_non_empty_with_annotations(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
@ -1050,6 +1031,90 @@ async def test_bedrock_validation_error_raises_directly(logging_obj: Logging):
assert getattr(excinfo.value, "status_code", None) == 400
_REFUSAL_PARTS: Final = ("I'm sorry", ", but I", " can't assist", " with that request.")
@pytest.mark.parametrize("async_stream", [False, True])
@pytest.mark.parametrize("transport", ["sdk", "http_handler"])
@pytest.mark.parametrize("finish_on_last_refusal", [False, True])
async def test_stream_forwards_refusal_only_deltas(
async_stream: bool, transport: str, finish_on_last_refusal: bool, logging_obj: Logging
) -> None:
last_index: Final = len(_REFUSAL_PARTS) - 1
refusal_choices: Final = tuple(
[
{
"index": 0,
"delta": {"refusal": part},
"finish_reason": "stop" if finish_on_last_refusal and index == last_index else None,
}
]
for index, part in enumerate(_REFUSAL_PARTS)
)
finish_choices: Final = () if finish_on_last_refusal else ([{"index": 0, "delta": {}, "finish_reason": "stop"}],)
body: Final = (
"".join(
"data: "
+ json.dumps(
{
"id": "chatcmpl-refusal",
"object": "chat.completion.chunk",
"created": 1,
"model": "test-model",
"choices": choices,
}
)
+ "\n\n"
for choices in (
[{"index": 0, "delta": {"role": "assistant", "content": None, "refusal": ""}, "finish_reason": None}],
*refusal_choices,
*finish_choices,
)
)
+ "data: [DONE]\n\n"
)
raw_response: Final = httpx.Response(
200,
content=body,
headers={"content-type": "text/event-stream"},
request=httpx.Request("POST", "https://provider.example/v1/chat/completions"),
)
client: Final = AsyncOpenAI(api_key="unused") if async_stream else OpenAI(api_key="unused")
stream: Final = (
(
AsyncStream(cast_to=ChatCompletionChunk, response=raw_response, client=client)
if isinstance(client, AsyncOpenAI)
else Stream(cast_to=ChatCompletionChunk, response=raw_response, client=client)
)
if transport == "sdk"
else OpenAIChatCompletionStreamingHandler(
streaming_response=raw_response.aiter_lines() if async_stream else raw_response.iter_lines(),
sync_stream=not async_stream,
)
)
response: Final = CustomStreamWrapper(
completion_stream=stream,
model="test-model",
logging_obj=logging_obj,
custom_llm_provider="openai" if transport == "sdk" else "hosted_vllm",
)
try:
chunks: Final = [chunk async for chunk in response] if async_stream else list(response)
finally:
if isinstance(client, AsyncOpenAI):
await client.close()
else:
client.close()
streamed_choices: Final = tuple(itertools.chain.from_iterable(chunk.choices for chunk in chunks))
assert "".join(getattr(choice.delta, "refusal", None) or "" for choice in streamed_choices) == "".join(
_REFUSAL_PARTS
)
assert "".join(choice.delta.content or "" for choice in streamed_choices) == ""
assert [choice.finish_reason for choice in streamed_choices if choice.finish_reason] == ["stop"]
def _hosted_vllm_stream_wrapper(logging_obj: Logging, error_payload: dict) -> CustomStreamWrapper:
"""A CustomStreamWrapper over the real OpenAI-compatible line iterator,
fed an HTTP 200 SSE body that carries an in-body error payload the way