mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-20 10:02:54 +00:00
Merge 662d5ba25f into 4710c3b30e
This commit is contained in:
commit
b893d091ef
6 changed files with 174 additions and 9 deletions
|
|
@ -406,6 +406,22 @@ class ChunkProcessor:
|
|||
None,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_native_finish_reason_from_chunks(chunks: Sequence["_BaseChunk"]) -> str | None:
|
||||
for chunk in reversed(chunks):
|
||||
choices = chunk.get("choices")
|
||||
if not choices:
|
||||
continue
|
||||
choice = choices[0]
|
||||
provider_specific_fields = getattr(choice, "provider_specific_fields", None)
|
||||
if provider_specific_fields is None and isinstance(choice, Mapping):
|
||||
provider_specific_fields = choice.get("provider_specific_fields")
|
||||
if provider_specific_fields and "native_finish_reason" in provider_specific_fields:
|
||||
native_finish_reason = provider_specific_fields["native_finish_reason"]
|
||||
if isinstance(native_finish_reason, str):
|
||||
return native_finish_reason
|
||||
return None
|
||||
|
||||
def build_base_response(self, chunks: Sequence["_BaseChunk"]) -> ModelResponse:
|
||||
chunk = self.first_chunk
|
||||
id: Final = ChunkProcessor._get_chunk_id(chunks)
|
||||
|
|
@ -429,6 +445,8 @@ class ChunkProcessor:
|
|||
if chunk_finish_reason is not None:
|
||||
finish_reason = chunk_finish_reason
|
||||
|
||||
native_finish_reason: Final = ChunkProcessor._get_native_finish_reason_from_chunks(chunks)
|
||||
|
||||
# Initialize the response dictionary
|
||||
response = ModelResponse(
|
||||
**{
|
||||
|
|
@ -452,6 +470,11 @@ class ChunkProcessor:
|
|||
"index": 0,
|
||||
"message": {"role": role, "content": ""},
|
||||
"finish_reason": finish_reason,
|
||||
**(
|
||||
{"provider_specific_fields": {"native_finish_reason": native_finish_reason}}
|
||||
if native_finish_reason is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
|
|
|
|||
|
|
@ -1262,10 +1262,14 @@ class CustomStreamWrapper:
|
|||
if _is_delta_empty:
|
||||
model_response.choices[0].delta = Delta(content=None) # ensure empty delta chunk returned
|
||||
# get any function call arguments
|
||||
model_response.choices[0].finish_reason = map_finish_reason(
|
||||
finish_reason=self.received_finish_reason
|
||||
) # ensure consistent output to openai
|
||||
|
||||
native_finish_reason: Final = self.received_finish_reason
|
||||
mapped_finish_reason: Final = map_finish_reason(finish_reason=self.received_finish_reason)
|
||||
model_response.choices[0].finish_reason = mapped_finish_reason # ensure consistent output to openai
|
||||
if native_finish_reason is not None and native_finish_reason != mapped_finish_reason:
|
||||
model_response.choices[0].provider_specific_fields = {
|
||||
**(model_response.choices[0].provider_specific_fields or {}),
|
||||
"native_finish_reason": native_finish_reason,
|
||||
}
|
||||
self.sent_last_chunk = True
|
||||
|
||||
return model_response
|
||||
|
|
|
|||
|
|
@ -2095,6 +2095,8 @@ class UnreportedUsage(Usage):
|
|||
|
||||
|
||||
class StreamingChoices(OpenAIObject):
|
||||
provider_specific_fields: dict[str, Any] | None = Field(default=None)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
finish_reason=None,
|
||||
|
|
@ -2107,11 +2109,14 @@ class StreamingChoices(OpenAIObject):
|
|||
# Fix Perplexity return both delta and message cause OpenWebUI repect text
|
||||
# https://github.com/BerriAI/litellm/issues/8455
|
||||
params.pop("message", None)
|
||||
mapped: Final = map_finish_reason(finish_reason) if finish_reason else None
|
||||
if mapped is not None and finish_reason != mapped:
|
||||
params["provider_specific_fields"] = {
|
||||
**(params.get("provider_specific_fields") or {}),
|
||||
"native_finish_reason": finish_reason,
|
||||
}
|
||||
super().__init__(**params)
|
||||
if finish_reason:
|
||||
self.finish_reason = map_finish_reason(finish_reason)
|
||||
else:
|
||||
self.finish_reason = None
|
||||
self.finish_reason = mapped
|
||||
self.index = index
|
||||
if delta is not None:
|
||||
if isinstance(delta, Delta):
|
||||
|
|
|
|||
|
|
@ -32,7 +32,15 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
unbind_budget_reservation_from_callbacks,
|
||||
)
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.types.utils import Delta, ImageResponse, ModelResponseStream, StreamingChoices, TranscriptionResponse
|
||||
from litellm.types.utils import (
|
||||
Choices,
|
||||
Delta,
|
||||
ImageResponse,
|
||||
Message,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
TranscriptionResponse,
|
||||
)
|
||||
|
||||
|
||||
def test_parent_otel_span_reads_dict_like_metadata_by_index():
|
||||
|
|
@ -371,6 +379,63 @@ class TestFinishReasonMapOutputsAreValid:
|
|||
)
|
||||
|
||||
|
||||
class TestStreamingChoicesNativeFinishReason:
|
||||
"""StreamingChoices must preserve the provider-native finish reason the same
|
||||
way Choices does, so streaming consumers can read the original provider value
|
||||
when map_finish_reason translates or defaults it."""
|
||||
|
||||
def test_unknown_provider_reason_is_preserved(self):
|
||||
choice: Final = StreamingChoices(finish_reason="network_error")
|
||||
|
||||
assert choice.finish_reason == "stop"
|
||||
assert choice.provider_specific_fields == {"native_finish_reason": "network_error"}
|
||||
|
||||
def test_translated_dialect_reason_is_preserved(self):
|
||||
choice: Final = StreamingChoices(finish_reason="end_turn")
|
||||
|
||||
assert choice.finish_reason == "stop"
|
||||
assert choice.provider_specific_fields == {"native_finish_reason": "end_turn"}
|
||||
|
||||
def test_openai_reason_is_not_stashed(self):
|
||||
choice: Final = StreamingChoices(finish_reason="stop")
|
||||
|
||||
assert choice.finish_reason == "stop"
|
||||
assert choice.provider_specific_fields is None
|
||||
|
||||
def test_none_finish_reason_keeps_provider_specific_fields_absent(self):
|
||||
choice: Final = StreamingChoices(finish_reason=None)
|
||||
|
||||
assert choice.finish_reason is None
|
||||
assert choice.provider_specific_fields is None
|
||||
|
||||
def test_matches_choices_behavior(self):
|
||||
streaming: Final = StreamingChoices(finish_reason="network_error")
|
||||
non_streaming: Final = Choices(
|
||||
finish_reason="network_error", index=0, message=Message(role="assistant", content="")
|
||||
)
|
||||
|
||||
assert streaming.finish_reason == non_streaming.finish_reason == "stop"
|
||||
assert streaming.provider_specific_fields == non_streaming.provider_specific_fields
|
||||
|
||||
def test_survives_proxy_chunk_serialization(self):
|
||||
"""The proxy serialises chunks with exclude_none and exclude_unset, the
|
||||
preserved value has to survive that round trip to reach the client."""
|
||||
chunk: Final = ModelResponseStream(
|
||||
id="chatcmpl-native-finish-reason",
|
||||
model="gpt-4o",
|
||||
choices=[
|
||||
StreamingChoices(index=0, delta=Delta(content=""), finish_reason="network_error")
|
||||
],
|
||||
)
|
||||
|
||||
payload: Final = json.loads(chunk.model_dump_json(exclude_none=True, exclude_unset=True))
|
||||
|
||||
assert payload["choices"][0]["finish_reason"] == "stop"
|
||||
assert payload["choices"][0]["provider_specific_fields"] == {
|
||||
"native_finish_reason": "network_error"
|
||||
}
|
||||
|
||||
|
||||
class TestRedactNestedMatchAndRegexKeys:
|
||||
def test_redacts_match_and_regex_recursively(self):
|
||||
payload = {
|
||||
|
|
|
|||
|
|
@ -2517,3 +2517,37 @@ def test_stream_chunk_builder_keeps_reasoning_items_on_text_only_replies():
|
|||
assert response is not None
|
||||
assert response.choices[0].message.content == "Hello world"
|
||||
assert response.choices[0].message.reasoning_items == reasoning_items
|
||||
|
||||
|
||||
def test_stream_chunk_builder_preserves_native_finish_reason_from_dict_chunks():
|
||||
"""SDK callers may pass model_dump()-ed chunks back to stream_chunk_builder, the
|
||||
preserved native finish reason has to survive that round trip too."""
|
||||
chunks: Final[list[dict[str, object]]] = [
|
||||
{
|
||||
"id": "chatcmpl-native-dict",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-5",
|
||||
"choices": [{"index": 0, "delta": {"content": "hi"}, "finish_reason": None}],
|
||||
},
|
||||
{
|
||||
"id": "chatcmpl-native-dict",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "glm-5",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {},
|
||||
"finish_reason": "stop",
|
||||
"provider_specific_fields": {"native_finish_reason": "network_error"},
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
response: Final = stream_chunk_builder(chunks=chunks)
|
||||
|
||||
assert response is not None
|
||||
assert response.choices[0].finish_reason == "stop"
|
||||
assert response.choices[0].provider_specific_fields == {"native_finish_reason": "network_error"}
|
||||
|
|
|
|||
|
|
@ -953,6 +953,40 @@ def test_logged_response_keeps_received_model_id_when_the_finish_chunk_is_synthe
|
|||
assert logged._hidden_params.get("received_model_id") == upstream_id
|
||||
|
||||
|
||||
def test_terminal_chunk_preserves_native_finish_reason(logging_obj: Logging):
|
||||
"""A finish-only chunk whose raw finish_reason is not an OpenAI reason must surface
|
||||
the provider value under provider_specific_fields.native_finish_reason while the
|
||||
OpenAI-facing finish_reason stays normalized."""
|
||||
finish_choice: Final = StreamingChoices(index=0, delta=Delta())
|
||||
finish_choice.finish_reason = "network_error"
|
||||
wrapper: Final = CustomStreamWrapper(
|
||||
completion_stream=ModelResponseListIterator(
|
||||
model_responses=[
|
||||
ModelResponseStream(
|
||||
id="chatcmpl-native-finish",
|
||||
model="glm-5",
|
||||
choices=[finish_choice],
|
||||
)
|
||||
]
|
||||
),
|
||||
model="glm-5",
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
emitted: Final = list(wrapper)
|
||||
|
||||
terminal: Final = [chunk for chunk in emitted if chunk.choices and chunk.choices[0].finish_reason is not None]
|
||||
assert len(terminal) == 1
|
||||
assert terminal[0].choices[0].finish_reason == "stop"
|
||||
assert terminal[0].choices[0].provider_specific_fields == {"native_finish_reason": "network_error"}
|
||||
|
||||
logged: Final = litellm.stream_chunk_builder(chunks=wrapper.chunks)
|
||||
assert logged is not None
|
||||
assert logged.choices[0].finish_reason == "stop"
|
||||
assert logged.choices[0].provider_specific_fields == {"native_finish_reason": "network_error"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_completion_start_time(logging_obj: Logging):
|
||||
"""Test that the start time is set correctly"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue