scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

Merge b2c75313e3 into 8ac9da84f7

Browse source
This commit is contained in:
David Koh 2026-11-06 02:00:31 +00:00 • committed by GitHub
commit d4f952db16
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 198 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -464,6 +464,13 @@ class ChunkProcessor:
None,
)
@staticmethod
def _choice_provider_specific_fields(chunk: "_BaseChunk") -> Mapping[str, object]:
choices: Final = chunk.get("choices")
if not choices:
return MappingProxyType({})
return object_mapping(_field_of(choices[0], "provider_specific_fields")) or MappingProxyType({})
def build_base_response(
self,
chunks: Sequence["_BaseChunk"],
@ -482,6 +489,10 @@ class ChunkProcessor:
role: Final = ChunkProcessor._get_role_from_chunks(chunks)
finish_reason: Final = ChunkProcessor._finish_reason_of_chunks(chunks)
choice_provider_specific_fields: Final[Mapping[str, builtins.object]] = dict(
chain.from_iterable(ChunkProcessor._choice_provider_specific_fields(chunk).items() for chunk in chunks)
)
# Initialize the response dictionary
response = ModelResponse(
**{
@ -508,6 +519,7 @@ class ChunkProcessor:
"index": 0,
"message": {"role": role, "content": ""},
"finish_reason": finish_reason,
"provider_specific_fields": choice_provider_specific_fields or None,
**_streamed_vllm_choice_fields(_first_choices(chunks)),
}
]

2
litellm/litellm_core_utils/streaming_handler.py
View file

@ -2108,7 +2108,7 @@ class CustomStreamWrapper:
{
**self.received_finish_choice_fields,
_upstream_choice_index(upstream_chunk): _upstream_choice_fields(
upstream_chunk, STREAM_FINISH_CHOICE_FIELDS
upstream_chunk, (*STREAM_FINISH_CHOICE_FIELDS, "provider_specific_fields")
),
}
)

9
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -3093,12 +3093,13 @@ class AmazonConverseConfig(BaseConfig):
## HANDLE TOOL CALLS
_message: Final = Message(**chat_completion_message)
mapped_finish_reason: Final = map_finish_reason(completion_response["stopReason"])
raw_finish_reason: Final = completion_response["stopReason"]
mapped_finish_reason: Final = map_finish_reason(raw_finish_reason)
initial_finish_reason: Final = (
"stop"
if resolved_json_mode and not filtered_tools and tools and mapped_finish_reason == "tool_calls"
else mapped_finish_reason
else raw_finish_reason
)
(
@ -3109,11 +3110,15 @@ class AmazonConverseConfig(BaseConfig):
tools=optional_params.get("tools"),
initial_finish_reason=initial_finish_reason,
)
choice_provider_specific_fields: Final = (
{"native_finish_reason": raw_finish_reason} if returned_finish_reason != raw_finish_reason else None
)
model_response.choices = [
litellm.Choices(
finish_reason=returned_finish_reason,
index=0,
message=returned_message,
provider_specific_fields=choice_provider_specific_fields,
)
]
model_response.created = int(time.time())

14
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -11,7 +11,7 @@ from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.caching.caching import InMemoryCache
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.core_helpers import as_str_mapping, map_finish_reason
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers
@ -711,6 +711,7 @@ class AWSEventStreamDecoder:
tool_use: ChatCompletionToolCallChunk | None = None
finish_reason = ""
usage: Usage | None = None
raw_stop_reason: Final = (as_str_mapping(chunk_data) or {}).get("stopReason")
provider_specific_fields: dict = {}
reasoning_content: str | None = None
thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None = None
@ -734,8 +735,8 @@ class AWSEventStreamDecoder:
) = self._handle_converse_delta_event(delta_obj, content_block_index)
elif "contentBlockIndex" in chunk_data: # stop block, no 'start' or 'delta' object
tool_use = self._handle_converse_stop_event(content_block_index)
elif "stopReason" in chunk_data:
finish_reason = map_finish_reason(chunk_data.get("stopReason", "stop"))
elif isinstance(raw_stop_reason, str):
finish_reason = map_finish_reason(raw_stop_reason)
self._provider_reasoning_tokens = AmazonConverseConfig.thinking_tokens_from_additional_fields(
chunk_data.get("additionalModelResponseFields")
)
@ -749,6 +750,12 @@ class AWSEventStreamDecoder:
self._thinking_ran = True
trace: Final = chunk_data.get("trace")
choice_provider_specific_fields: Final = (
{"native_finish_reason": raw_stop_reason}
if isinstance(raw_stop_reason, str) and raw_stop_reason != finish_reason
else None
)
carries_message_content: Final = bool(trace) or any(
key in chunk_data for key in ("start", "delta", "contentBlockIndex", "stopReason")
)
@ -759,6 +766,7 @@ class AWSEventStreamDecoder:
StreamingChoices(
finish_reason=finish_reason,
index=0, # Always 0 - Bedrock never returns multiple choices
provider_specific_fields=choice_provider_specific_fields,
delta=Delta(
content=text if carries_message_content else None,
role="assistant" if carries_message_content else None,

31
tests/unit/litellm_core_utils/test_streaming_handler.py
View file

@ -2367,6 +2367,37 @@ async def _collect(wrapper: CustomStreamWrapper, sync_mode: bool) -> list[ModelR
return [chunk async for chunk in wrapper]
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.parametrize("stop_reason", ["stop_sequence", "end_turn"])
@pytest.mark.asyncio
async def test_bedrock_stream_preserves_native_stop_reason_through_wrapper(
logging_obj: Logging, sync_mode: bool, stop_reason: str
) -> None:
from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder
decoder: Final = AWSEventStreamDecoder(model="bedrock/claude")
upstream: Final = (
decoder.converse_chunk_parser({"contentBlockIndex": 0, "delta": {"text": "done"}}),
decoder.converse_chunk_parser({"stopReason": stop_reason}),
)
wrapper: Final = CustomStreamWrapper(
completion_stream=ModelResponseListIterator(model_responses=list(upstream)),
model="bedrock/claude",
custom_llm_provider="bedrock",
logging_obj=logging_obj,
)
received: Final = await _collect(wrapper, sync_mode)
terminal: Final = next(chunk for chunk in received if chunk.choices and chunk.choices[0].finish_reason)
assert "".join(chunk.choices[0].delta.content or "" for chunk in received if chunk.choices) == "done"
assert terminal.choices[0].finish_reason == "stop"
assert terminal.choices[0].provider_specific_fields == {"native_finish_reason": stop_reason}
assembled: Final = litellm.stream_chunk_builder(chunks=received)
assert assembled is not None
assert assembled.choices[0].provider_specific_fields == {"native_finish_reason": stop_reason}
@pytest.mark.parametrize("custom_llm_provider", ["openai", "azure", "hosted_vllm"])
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio

136
tests/unit/llms/bedrock/chat/test_converse_transformation.py
View file

@ -29,6 +29,7 @@ from litellm.llms.anthropic.chat.transformation import (
)
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder
from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.types.llms.bedrock import ContentBlock, ConverseTokenUsageBlock, RequestObject
@ -5966,6 +5967,141 @@ def test_transform_response_native_structured_output():
assert result.choices[0].finish_reason == "stop"
# AWS Bedrock Converse stopReason values, accessed 2026-09-16:
# https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html
@pytest.mark.parametrize("stop_reason", ("stop_sequence", "end_turn"))
def test_transform_response_preserves_raw_bedrock_stop_reason(stop_reason: str):
response_json = {
"metrics": {"latencyMs": 1},
"output": {
"message": {
"role": "assistant",
"content": [{"text": "done"}],
}
},
"stopReason": stop_reason,
"usage": {
"inputTokens": 10,
"outputTokens": 1,
"totalTokens": 11,
},
}
raw_response = httpx.Response(
200,
json=response_json,
request=httpx.Request("POST", "https://bedrock.test/converse"),
)
result = AmazonConverseConfig().transform_response(
model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
raw_response=raw_response,
model_response=ModelResponse(),
logging_obj=None,
request_data={},
messages=[],
optional_params={},
litellm_params={},
encoding=None,
)
assert result.choices[0].finish_reason == "stop"
assert result.choices[0].provider_specific_fields == {
"native_finish_reason": stop_reason
}
def test_transform_response_preserves_raw_stop_reason_when_content_becomes_tool_call():
response_json = {
"metrics": {"latencyMs": 1},
"output": {
"message": {
"role": "assistant",
"content": [
{
"text": json.dumps(
{
"type": "function",
"name": "lookup_weather",
"parameters": {"city": "Seoul"},
}
)
}
],
}
},
"stopReason": "stop_sequence",
"usage": {
"inputTokens": 10,
"outputTokens": 1,
"totalTokens": 11,
},
}
raw_response = httpx.Response(
200,
json=response_json,
request=httpx.Request("POST", "https://bedrock.test/converse"),
)
result = AmazonConverseConfig().transform_response(
model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
raw_response=raw_response,
model_response=ModelResponse(),
logging_obj=None,
request_data={},
messages=[],
optional_params={
"tools": [
{
"type": "function",
"function": {
"name": "lookup_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
}
]
},
litellm_params={},
encoding=None,
)
assert result.choices[0].finish_reason == "tool_calls"
assert result.choices[0].provider_specific_fields == {
"native_finish_reason": "stop_sequence"
}
# AWS Bedrock Converse stopReason values, accessed 2026-09-16:
# https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html
@pytest.mark.parametrize("stop_reason", ("stop_sequence", "end_turn"))
def test_converse_chunk_parser_preserves_raw_bedrock_stop_reason(stop_reason: str):
result = AWSEventStreamDecoder(model="bedrock/claude").converse_chunk_parser(
{"stopReason": stop_reason}
)
assert result.choices[0].finish_reason == "stop"
assert result.choices[0].provider_specific_fields == {
"native_finish_reason": stop_reason
}
def test_converse_assembled_stream_preserves_raw_bedrock_stop_reason():
# AWS Bedrock Converse stopReason value, accessed 2026-09-16:
# https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html
result = AWSEventStreamDecoder(model="bedrock/claude").converse_chunk_parser(
{"stopReason": "stop_sequence"}
)
assembled = litellm.stream_chunk_builder(chunks=[result])
assert assembled is not None
assert assembled.choices[0].provider_specific_fields == {
"native_finish_reason": "stop_sequence"
}
def test_add_additional_properties_simple_object():
"""Object schemas without additionalProperties get it set to false."""
schema = {