From e50ae35ddec982ba51c3d3b5580caa38287ed4a3 Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 28 Oct 2026 08:06:56 +0000 Subject: [PATCH 1/5] test: cover JSON mode preamble text preservation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_bedrock_native_e2e.py | 94 ++++++++++++++++ .../test_anthropic_chat_transformation.py | 63 +++++++++++ .../chat/test_converse_transformation.py | 102 ++++++++++++++++++ 3 files changed, 259 insertions(+) diff --git a/tests/e2e/llm_translation/test_bedrock_native_e2e.py b/tests/e2e/llm_translation/test_bedrock_native_e2e.py index 23ff6ac612e..425d8b9b74a 100644 --- a/tests/e2e/llm_translation/test_bedrock_native_e2e.py +++ b/tests/e2e/llm_translation/test_bedrock_native_e2e.py @@ -9,6 +9,8 @@ from __future__ import annotations import pytest from typing import Final +from openai.types.chat import ChatCompletionMessage +from openai.types.shared_params.response_format_json_schema import ResponseFormatJSONSchema from pydantic import BaseModel, ConfigDict from e2e_config import unique_marker @@ -86,6 +88,55 @@ class InvokeResponse(BaseModel): content: list[ConverseContent] +class WarehouseAnswer(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + + answer: int + + +class JsonModePreambleFields(BaseModel): + model_config = ConfigDict(frozen=True) + + json_mode_preamble_text: str | None = None + + +class JsonModeAssistantMessage(BaseModel): + model_config = ConfigDict(frozen=True) + + content: str | None = None + provider_specific_fields: JsonModePreambleFields | None = None + + +BEDROCK_JSON_SCHEMA_BACKEND: Final = "bedrock/us.anthropic.claude-opus-5-5" +WAREHOUSE_PROMPT: Final = ( + "A warehouse has 3 shelves. Shelf A holds 17 boxes of 24 items, shelf B holds 9 boxes of 36 items minus 13 " + "damaged items, and shelf C holds twice as many items as shelf B. How many usable items are there in total? " + "Work through it carefully, then answer." +) +WAREHOUSE_RESPONSE_FORMAT: Final[ResponseFormatJSONSchema] = { + "type": "json_schema", + "json_schema": { + "name": "warehouse_answer", + "strict": True, + "schema": { + "type": "object", + "properties": {"answer": {"type": "integer"}}, + "required": ["answer"], + "additionalProperties": False, + }, + }, +} + + +def _warehouse_message_has_preamble(message: ChatCompletionMessage) -> bool: + structured_message: Final = JsonModeAssistantMessage.model_validate(message.model_dump()) + assert structured_message.content is not None + WarehouseAnswer.model_validate_json(structured_message.content) + provider_fields: Final = structured_message.provider_specific_fields + preamble: Final = provider_fields.json_mode_preamble_text if provider_fields is not None else None + return preamble is not None and bool(preamble.strip()) + + def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]: model = f"e2e-bedrock-native-{unique_marker()}" model_id = proxy.create_model( @@ -144,6 +195,49 @@ class TestBedrockNative: assert parsed.events assert all(event.name and event.date and event.participants for event in parsed.events) + @pytest.mark.provider_live + @pytest.mark.covers("llm.chat_completions.bedrock_converse.structured_output.preamble.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(BEDROCK_JSON_SCHEMA_BACKEND,), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) + ) + def test_bedrock_json_schema_response_exposes_text_before_json_tool_call( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model = f"e2e-bedrock-json-mode-preamble-{unique_marker()}" + model_id = proxy.create_model( + model, + LiteLLMParamsBody( + model=BEDROCK_JSON_SCHEMA_BACKEND, + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="us-east-1", + ), + ) + resources.defer(lambda: proxy.delete_model(model_id)) + key: Final = resources.key() + + assert any( + _warehouse_message_has_preamble( + sdk.openai(key) + .chat.completions.create( + model=model, + messages=[{"role": "user", "content": WAREHOUSE_PROMPT}], + response_format=WAREHOUSE_RESPONSE_FORMAT, + max_tokens=1500, + ) + .choices[0] + .message + ) + for _ in range(3) + ), "No model-generated preamble appeared in three Bedrock calls" + @pytest.mark.covers("llm.bedrock_native.bedrock_converse.basic.nonstream.works") @meta( Subject( diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index ef002a73e52..1c748f0c83d 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -9716,6 +9716,69 @@ def test_json_mode_keeps_the_thinking_that_preceded_the_json_tool_call(): assert [block["signature"] for block in message.thinking_blocks] == [thinking_block["signature"]] +def test_json_mode_preserves_nonblank_text_before_json_tool_call(): + preamble: Final = "Let me calculate the usable items. " + body: Final = _json_mode_message_body("tool_use", None) + result: Final = _transform_json_mode_body( + {**body, "content": [{"type": "text", "text": preamble}, *body["content"]]} + ) + + message: Final = result.choices[0].message + assert message.content == '{"answer": "Hello"}' + assert message.provider_specific_fields["json_mode_preamble_text"] == preamble + + +def test_json_mode_preamble_preserves_refusal_provider_fields(): + preamble: Final = "Let me explain the result. " + stop_details: Final = ANTHROPIC_DOCS_REFUSAL_MESSAGE["stop_details"] + body: Final = _json_mode_message_body("refusal", stop_details) + result: Final = _transform_json_mode_body( + {**body, "content": [{"type": "text", "text": preamble}, *body["content"]]} + ) + + message: Final = result.choices[0].message + assert message.content == '{"answer": "Hello"}' + assert message.provider_specific_fields == { + "stop_reason": "refusal", + "stop_details": stop_details, + "json_mode_preamble_text": preamble, + } + + +def test_json_mode_text_without_tool_call_is_unchanged(): + body: Final = _json_mode_message_body("tool_use", None) + result: Final = _transform_json_mode_body( + {**body, "content": [{"type": "text", "text": "A plain response."}]} + ) + + message: Final = result.choices[0].message + assert message.content == "A plain response." + assert message.tool_calls is None + assert "json_mode_preamble_text" not in (message.provider_specific_fields or {}) + + +def test_json_mode_with_a_real_tool_keeps_mixed_response_behavior(): + preamble: Final = "I will look up the details. " + body: Final = _json_mode_message_body("tool_use", None) + user_tool_call: Final = { + "type": "tool_use", + "id": "toolu_user", + "name": "get_weather", + "input": {"location": "San Francisco"}, + } + result: Final = _transform_json_mode_body( + {**body, "content": [{"type": "text", "text": preamble}, *body["content"], user_tool_call]} + ) + + message: Final = result.choices[0].message + assert message.content == f'{preamble}{{"answer": "Hello"}}' + assert message.tool_calls is not None + assert len(message.tool_calls) == 1 + assert message.tool_calls[0].function.name == "get_weather" + assert message.tool_calls[0].function.arguments == '{"location": "San Francisco"}' + assert "json_mode_preamble_text" not in (message.provider_specific_fields or {}) + + def test_non_dict_upstream_body_still_transforms_without_refusal_fields() -> None: result = AnthropicConfig().transform_parsed_response( completion_response=MagicMock(), diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 154797bab5a..f89ca376e47 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -6736,6 +6736,108 @@ def test_transform_response_finish_reason_stop_when_json_mode_filters_all_tools( assert result.choices[0].finish_reason == "stop" +def _transform_json_mode_content_blocks(content: list[dict[str, object]]) -> ModelResponse: + response_json: Final = { + "metrics": {"latencyMs": 100}, + "output": {"message": {"role": "assistant", "content": content}}, + "stopReason": "tool_use", + "usage": { + "inputTokens": 50, + "outputTokens": 20, + "totalTokens": 70, + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + }, + } + + class MockResponse: + def json(self) -> dict[str, object]: + return response_json + + @property + def text(self) -> str: + return json.dumps(response_json) + + return AmazonConverseConfig()._transform_response( + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + response=MockResponse(), + model_response=ModelResponse(), + stream=False, + logging_obj=None, + optional_params={"json_mode": True}, + api_key=None, + data=None, + messages=[], + encoding=None, + ) + + +def test_bedrock_json_mode_preserves_nonblank_text_before_json_tool_call(): + preamble: Final = "Let me calculate the usable items. " + result: Final = _transform_json_mode_content_blocks( + [ + {"text": preamble}, + { + "toolUse": { + "toolUseId": "tooluse_json_001", + "name": "json_tool_call", + "input": {"answer": 42}, + } + }, + ] + ) + + message: Final = result.choices[0].message + assert message.content == '{"answer": 42}' + assert message.provider_specific_fields["json_mode_preamble_text"] == preamble + + +def test_bedrock_json_mode_omits_whitespace_only_preamble(): + result: Final = _transform_json_mode_content_blocks( + [ + {"text": " \n\t"}, + { + "toolUse": { + "toolUseId": "tooluse_json_001", + "name": "json_tool_call", + "input": {"answer": 42}, + } + }, + ] + ) + + message: Final = result.choices[0].message + assert message.content == '{"answer": 42}' + assert "json_mode_preamble_text" not in (message.provider_specific_fields or {}) + + +def test_bedrock_json_mode_preamble_preserves_reasoning_content_fields(): + reasoning_content: Final = { + "reasoningText": {"text": "The answer is 42.", "signature": "signature-1"} + } + preamble: Final = "Here is the result. " + result: Final = _transform_json_mode_content_blocks( + [ + {"reasoningContent": reasoning_content}, + {"text": preamble}, + { + "toolUse": { + "toolUseId": "tooluse_json_001", + "name": "json_tool_call", + "input": {"answer": 42}, + } + }, + ] + ) + + message: Final = result.choices[0].message + assert message.content == '{"answer": 42}' + assert message.provider_specific_fields["json_mode_preamble_text"] == preamble + assert message.provider_specific_fields["reasoningContentBlocks"] == [reasoning_content] + + def test_transform_response_json_mode_truncated_tool_call_keeps_length_finish_reason(): """ When json_mode filters out the synthetic json_tool_call but Bedrock From e323ea243a11830677e2015c9390b3912af78457 Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 28 Oct 2026 08:37:57 +0000 Subject: [PATCH 2/5] fix: preserve json mode preamble text Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/llms/anthropic/chat/transformation.py | 8 +++++++- litellm/llms/bedrock/chat/converse_transformation.py | 11 +++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/litellm/constants.py b/litellm/constants.py index 8cc51d0ab5e..8a2e69e84d1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1633,6 +1633,7 @@ OPENAI_FINISH_REASONS: Final = [ ] HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = get_env_int("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60) # 1 minute RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call +JSON_MODE_PREAMBLE_TEXT_KEY: Final = "json_mode_preamble_text" ########################### Logging Callback Constants ########################### AZURE_STORAGE_MSFT_VERSION: Final = "2019-07-07" diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 92e7c84031c..409fc963553 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -21,6 +21,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, + JSON_MODE_PREAMBLE_TEXT_KEY, RESPONSE_FORMAT_TOOL_NAME, ) from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason @@ -3056,7 +3057,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): REFUSAL_STOP_FIELDS.validate_python(completion_response) ) completion_response["stop_reason"] = "stop" - if refusal_fields: + if text_content and text_content.strip(): + json_mode_message.provider_specific_fields = { + **refusal_fields, + JSON_MODE_PREAMBLE_TEXT_KEY: text_content, + } + elif refusal_fields: json_mode_message.provider_specific_fields = dict(refusal_fields) if thinking_blocks: json_mode_message.thinking_blocks = thinking_blocks diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index f54c352fd6e..63c6d8a252b 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -19,6 +19,7 @@ import litellm from litellm._logging import verbose_logger from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, + JSON_MODE_PREAMBLE_TEXT_KEY, RESPONSE_FORMAT_TOOL_NAME, ) from litellm.litellm_core_utils.core_helpers import ( @@ -2676,6 +2677,16 @@ class AmazonConverseConfig(BaseConfig): json_mode_content_str: str | None = tools[0]["function"].get("arguments") if json_mode_content_str is not None: json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties(json_mode_content_str) + message_fields: Final = as_str_mapping(chat_completion_message) + existing_content: Final = (message_fields or {}).get("content") + if isinstance(existing_content, str) and existing_content.strip(): + existing_provider_fields: Final = TypeAdapter(dict[str, object]).validate_python( + (message_fields.get("provider_specific_fields") or {}) if message_fields is not None else {} + ) + chat_completion_message["provider_specific_fields"] = { # rebind-ok: response is built in place + **existing_provider_fields, + JSON_MODE_PREAMBLE_TEXT_KEY: existing_content, + } chat_completion_message["content"] = json_mode_content_str return None From 814fd2275e65d21bac2a04b96f66cc12884f60b9 Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 28 Oct 2026 08:50:25 +0000 Subject: [PATCH 3/5] test: strengthen JSON preamble mutation coverage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_bedrock_native_e2e.py | 4 +-- .../test_anthropic_chat_transformation.py | 15 +++++++++++ .../chat/test_converse_transformation.py | 26 +++++++++++++++---- 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/tests/e2e/llm_translation/test_bedrock_native_e2e.py b/tests/e2e/llm_translation/test_bedrock_native_e2e.py index 425d8b9b74a..95720bcace2 100644 --- a/tests/e2e/llm_translation/test_bedrock_native_e2e.py +++ b/tests/e2e/llm_translation/test_bedrock_native_e2e.py @@ -210,8 +210,8 @@ class TestBedrockNative: def test_bedrock_json_schema_response_exposes_text_before_json_tool_call( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = f"e2e-bedrock-json-mode-preamble-{unique_marker()}" - model_id = proxy.create_model( + model: Final = f"e2e-bedrock-json-mode-preamble-{unique_marker()}" + model_id: Final = proxy.create_model( model, LiteLLMParamsBody( model=BEDROCK_JSON_SCHEMA_BACKEND, diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 1c748f0c83d..09933605ade 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -9756,6 +9756,14 @@ def test_json_mode_text_without_tool_call_is_unchanged(): assert message.tool_calls is None assert "json_mode_preamble_text" not in (message.provider_specific_fields or {}) + preamble: Final = "Let me answer the question. " + json_result: Final = _transform_json_mode_body( + {**body, "content": [{"type": "text", "text": preamble}, *body["content"]]} + ) + json_message: Final = json_result.choices[0].message + assert json_message.content == '{"answer": "Hello"}' + assert json_message.provider_specific_fields["json_mode_preamble_text"] == preamble + def test_json_mode_with_a_real_tool_keeps_mixed_response_behavior(): preamble: Final = "I will look up the details. " @@ -9778,6 +9786,13 @@ def test_json_mode_with_a_real_tool_keeps_mixed_response_behavior(): assert message.tool_calls[0].function.arguments == '{"location": "San Francisco"}' assert "json_mode_preamble_text" not in (message.provider_specific_fields or {}) + json_result: Final = _transform_json_mode_body( + {**body, "content": [{"type": "text", "text": preamble}, *body["content"]]} + ) + json_message: Final = json_result.choices[0].message + assert json_message.content == '{"answer": "Hello"}' + assert json_message.provider_specific_fields["json_mode_preamble_text"] == preamble + def test_non_dict_upstream_body_still_transforms_without_refusal_fields() -> None: result = AnthropicConfig().transform_parsed_response( diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index f89ca376e47..fcdddb6daac 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -6761,7 +6761,7 @@ def _transform_json_mode_content_blocks(content: list[dict[str, object]]) -> Mod return json.dumps(response_json) return AmazonConverseConfig()._transform_response( - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + model="bedrock/us.anthropic.claude-haiku-5-5", response=MockResponse(), model_response=ModelResponse(), stream=False, @@ -6795,7 +6795,7 @@ def test_bedrock_json_mode_preserves_nonblank_text_before_json_tool_call(): def test_bedrock_json_mode_omits_whitespace_only_preamble(): - result: Final = _transform_json_mode_content_blocks( + whitespace_result: Final = _transform_json_mode_content_blocks( [ {"text": " \n\t"}, { @@ -6808,9 +6808,25 @@ def test_bedrock_json_mode_omits_whitespace_only_preamble(): ] ) - message: Final = result.choices[0].message - assert message.content == '{"answer": 42}' - assert "json_mode_preamble_text" not in (message.provider_specific_fields or {}) + whitespace_message: Final = whitespace_result.choices[0].message + assert whitespace_message.content == '{"answer": 42}' + assert "json_mode_preamble_text" not in (whitespace_message.provider_specific_fields or {}) + + preamble: Final = "Here is the answer. " + nonblank_result: Final = _transform_json_mode_content_blocks( + [ + {"text": preamble}, + { + "toolUse": { + "toolUseId": "tooluse_json_001", + "name": "json_tool_call", + "input": {"answer": 42}, + } + }, + ] + ) + nonblank_message: Final = nonblank_result.choices[0].message + assert nonblank_message.provider_specific_fields["json_mode_preamble_text"] == preamble def test_bedrock_json_mode_preamble_preserves_reasoning_content_fields(): From 8fa1ed4066f857555f9f73cade43708ee12a3ab1 Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 28 Oct 2026 08:57:00 +0000 Subject: [PATCH 4/5] test: drop duplicate guard assertions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_anthropic_chat_transformation.py | 15 ------------ .../chat/test_converse_transformation.py | 24 ++++--------------- 2 files changed, 4 insertions(+), 35 deletions(-) diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 09933605ade..1c748f0c83d 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -9756,14 +9756,6 @@ def test_json_mode_text_without_tool_call_is_unchanged(): assert message.tool_calls is None assert "json_mode_preamble_text" not in (message.provider_specific_fields or {}) - preamble: Final = "Let me answer the question. " - json_result: Final = _transform_json_mode_body( - {**body, "content": [{"type": "text", "text": preamble}, *body["content"]]} - ) - json_message: Final = json_result.choices[0].message - assert json_message.content == '{"answer": "Hello"}' - assert json_message.provider_specific_fields["json_mode_preamble_text"] == preamble - def test_json_mode_with_a_real_tool_keeps_mixed_response_behavior(): preamble: Final = "I will look up the details. " @@ -9786,13 +9778,6 @@ def test_json_mode_with_a_real_tool_keeps_mixed_response_behavior(): assert message.tool_calls[0].function.arguments == '{"location": "San Francisco"}' assert "json_mode_preamble_text" not in (message.provider_specific_fields or {}) - json_result: Final = _transform_json_mode_body( - {**body, "content": [{"type": "text", "text": preamble}, *body["content"]]} - ) - json_message: Final = json_result.choices[0].message - assert json_message.content == '{"answer": "Hello"}' - assert json_message.provider_specific_fields["json_mode_preamble_text"] == preamble - def test_non_dict_upstream_body_still_transforms_without_refusal_fields() -> None: result = AnthropicConfig().transform_parsed_response( diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index fcdddb6daac..143f0981814 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -6795,7 +6795,7 @@ def test_bedrock_json_mode_preserves_nonblank_text_before_json_tool_call(): def test_bedrock_json_mode_omits_whitespace_only_preamble(): - whitespace_result: Final = _transform_json_mode_content_blocks( + result: Final = _transform_json_mode_content_blocks( [ {"text": " \n\t"}, { @@ -6808,25 +6808,9 @@ def test_bedrock_json_mode_omits_whitespace_only_preamble(): ] ) - whitespace_message: Final = whitespace_result.choices[0].message - assert whitespace_message.content == '{"answer": 42}' - assert "json_mode_preamble_text" not in (whitespace_message.provider_specific_fields or {}) - - preamble: Final = "Here is the answer. " - nonblank_result: Final = _transform_json_mode_content_blocks( - [ - {"text": preamble}, - { - "toolUse": { - "toolUseId": "tooluse_json_001", - "name": "json_tool_call", - "input": {"answer": 42}, - } - }, - ] - ) - nonblank_message: Final = nonblank_result.choices[0].message - assert nonblank_message.provider_specific_fields["json_mode_preamble_text"] == preamble + message: Final = result.choices[0].message + assert message.content == '{"answer": 42}' + assert "json_mode_preamble_text" not in (message.provider_specific_fields or {}) def test_bedrock_json_mode_preamble_preserves_reasoning_content_fields(): From 9a2d61680ca384267cb8fc5eaa00913ec4b7531c Mon Sep 17 00:00:00 2001 From: ryan Date: Thu, 29 Oct 2026 06:22:14 +0000 Subject: [PATCH 5/5] fix: reuse Bedrock JSON adapter and test streamed preambles Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../bedrock/chat/converse_transformation.py | 2 +- .../chat/test_converse_transformation.py | 68 ++++++++++++++++++- 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 63c6d8a252b..fd97f186db7 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -2680,7 +2680,7 @@ class AmazonConverseConfig(BaseConfig): message_fields: Final = as_str_mapping(chat_completion_message) existing_content: Final = (message_fields or {}).get("content") if isinstance(existing_content, str) and existing_content.strip(): - existing_provider_fields: Final = TypeAdapter(dict[str, object]).validate_python( + existing_provider_fields: Final = _JSON_OBJECT.validate_python( (message_fields.get("provider_specific_fields") or {}) if message_fields is not None else {} ) chat_completion_message["provider_specific_fields"] = { # rebind-ok: response is built in place diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 143f0981814..06458067995 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -3,13 +3,14 @@ import copy import json import os import re -from collections.abc import Iterator, Mapping -from typing import Final +from collections.abc import Iterable, Iterator, Mapping +from typing import Final, cast from unittest.mock import MagicMock, Mock, patch import httpx import pytest import respx +from pydantic import TypeAdapter import litellm from litellm import ModelResponse, Router, completion @@ -23,12 +24,13 @@ from litellm.llms.anthropic.chat.transformation import ( NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS, NATIVE_OUTPUT_MAX_UNION_PARAMETERS, ) -from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig +from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.llms.bedrock import ContentBlock, ConverseTokenUsageBlock, RequestObject from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ModelResponseStream def test_transform_usage(): @@ -6794,6 +6796,66 @@ def test_bedrock_json_mode_preserves_nonblank_text_before_json_tool_call(): assert message.provider_specific_fields["json_mode_preamble_text"] == preamble +def test_bedrock_json_mode_fake_stream_preserves_preamble_in_chunks() -> None: + preamble: Final = "Let me calculate the usable items. " + response_json: Final = { + "output": { + "message": { + "role": "assistant", + "content": [ + {"text": preamble}, + { + "toolUse": { + "toolUseId": "tooluse_json_001", + "name": "json_tool_call", + "input": {"answer": 42}, + } + }, + ], + } + }, + "stopReason": "tool_use", + "usage": {"inputTokens": 50, "outputTokens": 20, "totalTokens": 70}, + } + + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=response_json, request=request) + + model: Final = "bedrock/us.anthropic.claude-opus-5-5" + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + stream: Final = completion( + model=model, + messages=[{"role": "user", "content": "How many usable items are there?"}], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": { + "type": "object", + "properties": {"answer": {"type": "integer"}}, + "required": ["answer"], + "additionalProperties": False, + }, + }, + }, + stream=True, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + client=client, + ) + chunks: Final = list(cast(Iterable[ModelResponseStream], stream)) + + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) == '{"answer": 42}' + provider_fields_adapter: Final = TypeAdapter(Mapping[str, object]) + provider_fields: Final = tuple( + provider_fields_adapter.validate_python(chunk.choices[0].delta.provider_specific_fields or {}) + for chunk in chunks + if chunk.choices + ) + assert any(fields.get("json_mode_preamble_text") == preamble for fields in provider_fields) + + def test_bedrock_json_mode_omits_whitespace_only_preamble(): result: Final = _transform_json_mode_content_blocks( [