diff --git a/litellm/constants.py b/litellm/constants.py index c17e776f54b..f7e928a783f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1657,6 +1657,7 @@ OPENAI_FINISH_REASONS: Final = [ ] HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = get_env_int("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60) # 1 minute RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call +JSON_MODE_PREAMBLE_TEXT_KEY: Final = "json_mode_preamble_text" ########################### Logging Callback Constants ########################### AZURE_STORAGE_MSFT_VERSION: Final = "2019-07-07" diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index d773e31c733..cbd256a4861 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -22,6 +22,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, + JSON_MODE_PREAMBLE_TEXT_KEY, RESPONSE_FORMAT_TOOL_NAME, ) from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason @@ -3059,7 +3060,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): REFUSAL_STOP_FIELDS.validate_python(completion_response) ) completion_response["stop_reason"] = "stop" - if refusal_fields: + if text_content and text_content.strip(): + json_mode_message.provider_specific_fields = { + **refusal_fields, + JSON_MODE_PREAMBLE_TEXT_KEY: text_content, + } + elif refusal_fields: json_mode_message.provider_specific_fields = dict(refusal_fields) if thinking_blocks: json_mode_message.thinking_blocks = thinking_blocks diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 3800489ec05..7976e0f5278 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -21,6 +21,7 @@ from litellm.anthropic_beta_gated_fields import active_beta_values, log_beta_gat from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, + JSON_MODE_PREAMBLE_TEXT_KEY, RESPONSE_FORMAT_TOOL_NAME, ) from litellm.litellm_core_utils.core_helpers import ( @@ -2708,6 +2709,16 @@ class AmazonConverseConfig(BaseConfig): json_mode_content_str: str | None = tools[0]["function"].get("arguments") if json_mode_content_str is not None: json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties(json_mode_content_str) + message_fields: Final = as_str_mapping(chat_completion_message) + existing_content: Final = (message_fields or {}).get("content") + if isinstance(existing_content, str) and existing_content.strip(): + existing_provider_fields: Final = _JSON_OBJECT.validate_python( + (message_fields.get("provider_specific_fields") or {}) if message_fields is not None else {} + ) + chat_completion_message["provider_specific_fields"] = { # rebind-ok: response is built in place + **existing_provider_fields, + JSON_MODE_PREAMBLE_TEXT_KEY: existing_content, + } chat_completion_message["content"] = json_mode_content_str return None diff --git a/tests/e2e/llm_translation/test_bedrock_native_e2e.py b/tests/e2e/llm_translation/test_bedrock_native_e2e.py index 23ff6ac612e..95720bcace2 100644 --- a/tests/e2e/llm_translation/test_bedrock_native_e2e.py +++ b/tests/e2e/llm_translation/test_bedrock_native_e2e.py @@ -9,6 +9,8 @@ from __future__ import annotations import pytest from typing import Final +from openai.types.chat import ChatCompletionMessage +from openai.types.shared_params.response_format_json_schema import ResponseFormatJSONSchema from pydantic import BaseModel, ConfigDict from e2e_config import unique_marker @@ -86,6 +88,55 @@ class InvokeResponse(BaseModel): content: list[ConverseContent] +class WarehouseAnswer(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", strict=True) + + answer: int + + +class JsonModePreambleFields(BaseModel): + model_config = ConfigDict(frozen=True) + + json_mode_preamble_text: str | None = None + + +class JsonModeAssistantMessage(BaseModel): + model_config = ConfigDict(frozen=True) + + content: str | None = None + provider_specific_fields: JsonModePreambleFields | None = None + + +BEDROCK_JSON_SCHEMA_BACKEND: Final = "bedrock/us.anthropic.claude-opus-5-5" +WAREHOUSE_PROMPT: Final = ( + "A warehouse has 3 shelves. Shelf A holds 17 boxes of 24 items, shelf B holds 9 boxes of 36 items minus 13 " + "damaged items, and shelf C holds twice as many items as shelf B. How many usable items are there in total? " + "Work through it carefully, then answer." +) +WAREHOUSE_RESPONSE_FORMAT: Final[ResponseFormatJSONSchema] = { + "type": "json_schema", + "json_schema": { + "name": "warehouse_answer", + "strict": True, + "schema": { + "type": "object", + "properties": {"answer": {"type": "integer"}}, + "required": ["answer"], + "additionalProperties": False, + }, + }, +} + + +def _warehouse_message_has_preamble(message: ChatCompletionMessage) -> bool: + structured_message: Final = JsonModeAssistantMessage.model_validate(message.model_dump()) + assert structured_message.content is not None + WarehouseAnswer.model_validate_json(structured_message.content) + provider_fields: Final = structured_message.provider_specific_fields + preamble: Final = provider_fields.json_mode_preamble_text if provider_fields is not None else None + return preamble is not None and bool(preamble.strip()) + + def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]: model = f"e2e-bedrock-native-{unique_marker()}" model_id = proxy.create_model( @@ -144,6 +195,49 @@ class TestBedrockNative: assert parsed.events assert all(event.name and event.date and event.participants for event in parsed.events) + @pytest.mark.provider_live + @pytest.mark.covers("llm.chat_completions.bedrock_converse.structured_output.preamble.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(BEDROCK_JSON_SCHEMA_BACKEND,), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) + ) + def test_bedrock_json_schema_response_exposes_text_before_json_tool_call( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = f"e2e-bedrock-json-mode-preamble-{unique_marker()}" + model_id: Final = proxy.create_model( + model, + LiteLLMParamsBody( + model=BEDROCK_JSON_SCHEMA_BACKEND, + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="us-east-1", + ), + ) + resources.defer(lambda: proxy.delete_model(model_id)) + key: Final = resources.key() + + assert any( + _warehouse_message_has_preamble( + sdk.openai(key) + .chat.completions.create( + model=model, + messages=[{"role": "user", "content": WAREHOUSE_PROMPT}], + response_format=WAREHOUSE_RESPONSE_FORMAT, + max_tokens=1500, + ) + .choices[0] + .message + ) + for _ in range(3) + ), "No model-generated preamble appeared in three Bedrock calls" + @pytest.mark.covers("llm.bedrock_native.bedrock_converse.basic.nonstream.works") @meta( Subject( diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index cbb60caa57b..3b70ba1f760 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -9761,6 +9761,69 @@ def test_json_mode_keeps_the_thinking_that_preceded_the_json_tool_call(): assert [block["signature"] for block in message.thinking_blocks] == [thinking_block["signature"]] +def test_json_mode_preserves_nonblank_text_before_json_tool_call(): + preamble: Final = "Let me calculate the usable items. " + body: Final = _json_mode_message_body("tool_use", None) + result: Final = _transform_json_mode_body( + {**body, "content": [{"type": "text", "text": preamble}, *body["content"]]} + ) + + message: Final = result.choices[0].message + assert message.content == '{"answer": "Hello"}' + assert message.provider_specific_fields["json_mode_preamble_text"] == preamble + + +def test_json_mode_preamble_preserves_refusal_provider_fields(): + preamble: Final = "Let me explain the result. " + stop_details: Final = ANTHROPIC_DOCS_REFUSAL_MESSAGE["stop_details"] + body: Final = _json_mode_message_body("refusal", stop_details) + result: Final = _transform_json_mode_body( + {**body, "content": [{"type": "text", "text": preamble}, *body["content"]]} + ) + + message: Final = result.choices[0].message + assert message.content == '{"answer": "Hello"}' + assert message.provider_specific_fields == { + "stop_reason": "refusal", + "stop_details": stop_details, + "json_mode_preamble_text": preamble, + } + + +def test_json_mode_text_without_tool_call_is_unchanged(): + body: Final = _json_mode_message_body("tool_use", None) + result: Final = _transform_json_mode_body( + {**body, "content": [{"type": "text", "text": "A plain response."}]} + ) + + message: Final = result.choices[0].message + assert message.content == "A plain response." + assert message.tool_calls is None + assert "json_mode_preamble_text" not in (message.provider_specific_fields or {}) + + +def test_json_mode_with_a_real_tool_keeps_mixed_response_behavior(): + preamble: Final = "I will look up the details. " + body: Final = _json_mode_message_body("tool_use", None) + user_tool_call: Final = { + "type": "tool_use", + "id": "toolu_user", + "name": "get_weather", + "input": {"location": "San Francisco"}, + } + result: Final = _transform_json_mode_body( + {**body, "content": [{"type": "text", "text": preamble}, *body["content"], user_tool_call]} + ) + + message: Final = result.choices[0].message + assert message.content == f'{preamble}{{"answer": "Hello"}}' + assert message.tool_calls is not None + assert len(message.tool_calls) == 1 + assert message.tool_calls[0].function.name == "get_weather" + assert message.tool_calls[0].function.arguments == '{"location": "San Francisco"}' + assert "json_mode_preamble_text" not in (message.provider_specific_fields or {}) + + def test_non_dict_upstream_body_still_transforms_without_refusal_fields() -> None: result = AnthropicConfig().transform_parsed_response( completion_response=MagicMock(), diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 524caaba7a1..875a06a593a 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -4,13 +4,14 @@ import json import logging import os import re -from collections.abc import Iterator, Mapping -from typing import Final +from collections.abc import Iterable, Iterator, Mapping +from typing import Final, cast from unittest.mock import MagicMock, Mock, patch import httpx import pytest import respx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger @@ -31,6 +32,7 @@ from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTI from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.llms.bedrock import ContentBlock, ConverseTokenUsageBlock, RequestObject from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ModelResponseStream def test_transform_usage(): @@ -6834,6 +6836,168 @@ def test_transform_response_finish_reason_stop_when_json_mode_filters_all_tools( assert result.choices[0].finish_reason == "stop" +def _transform_json_mode_content_blocks(content: list[dict[str, object]]) -> ModelResponse: + response_json: Final = { + "metrics": {"latencyMs": 100}, + "output": {"message": {"role": "assistant", "content": content}}, + "stopReason": "tool_use", + "usage": { + "inputTokens": 50, + "outputTokens": 20, + "totalTokens": 70, + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + }, + } + + class MockResponse: + def json(self) -> dict[str, object]: + return response_json + + @property + def text(self) -> str: + return json.dumps(response_json) + + return AmazonConverseConfig()._transform_response( + model="bedrock/us.anthropic.claude-haiku-5-5", + response=MockResponse(), + model_response=ModelResponse(), + stream=False, + logging_obj=None, + optional_params={"json_mode": True}, + api_key=None, + data=None, + messages=[], + encoding=None, + ) + + +def test_bedrock_json_mode_preserves_nonblank_text_before_json_tool_call(): + preamble: Final = "Let me calculate the usable items. " + result: Final = _transform_json_mode_content_blocks( + [ + {"text": preamble}, + { + "toolUse": { + "toolUseId": "tooluse_json_001", + "name": "json_tool_call", + "input": {"answer": 42}, + } + }, + ] + ) + + message: Final = result.choices[0].message + assert message.content == '{"answer": 42}' + assert message.provider_specific_fields["json_mode_preamble_text"] == preamble + + +def test_bedrock_json_mode_fake_stream_preserves_preamble_in_chunks() -> None: + preamble: Final = "Let me calculate the usable items. " + response_json: Final = { + "output": { + "message": { + "role": "assistant", + "content": [ + {"text": preamble}, + { + "toolUse": { + "toolUseId": "tooluse_json_001", + "name": "json_tool_call", + "input": {"answer": 42}, + } + }, + ], + } + }, + "stopReason": "tool_use", + "usage": {"inputTokens": 50, "outputTokens": 20, "totalTokens": 70}, + } + + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=response_json, request=request) + + model: Final = "bedrock/us.anthropic.claude-opus-5-5" + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + stream: Final = completion( + model=model, + messages=[{"role": "user", "content": "How many usable items are there?"}], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": { + "type": "object", + "properties": {"answer": {"type": "integer"}}, + "required": ["answer"], + "additionalProperties": False, + }, + }, + }, + stream=True, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + client=client, + ) + chunks: Final = list(cast(Iterable[ModelResponseStream], stream)) + + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) == '{"answer": 42}' + provider_fields_adapter: Final = TypeAdapter(Mapping[str, object]) + provider_fields: Final = tuple( + provider_fields_adapter.validate_python(chunk.choices[0].delta.provider_specific_fields or {}) + for chunk in chunks + if chunk.choices + ) + assert any(fields.get("json_mode_preamble_text") == preamble for fields in provider_fields) + + +def test_bedrock_json_mode_omits_whitespace_only_preamble(): + result: Final = _transform_json_mode_content_blocks( + [ + {"text": " \n\t"}, + { + "toolUse": { + "toolUseId": "tooluse_json_001", + "name": "json_tool_call", + "input": {"answer": 42}, + } + }, + ] + ) + + message: Final = result.choices[0].message + assert message.content == '{"answer": 42}' + assert "json_mode_preamble_text" not in (message.provider_specific_fields or {}) + + +def test_bedrock_json_mode_preamble_preserves_reasoning_content_fields(): + reasoning_content: Final = { + "reasoningText": {"text": "The answer is 42.", "signature": "signature-1"} + } + preamble: Final = "Here is the result. " + result: Final = _transform_json_mode_content_blocks( + [ + {"reasoningContent": reasoning_content}, + {"text": preamble}, + { + "toolUse": { + "toolUseId": "tooluse_json_001", + "name": "json_tool_call", + "input": {"answer": 42}, + } + }, + ] + ) + + message: Final = result.choices[0].message + assert message.content == '{"answer": 42}' + assert message.provider_specific_fields["json_mode_preamble_text"] == preamble + assert message.provider_specific_fields["reasoningContentBlocks"] == [reasoning_content] + + def test_transform_response_json_mode_truncated_tool_call_keeps_length_finish_reason(): """ When json_mode filters out the synthetic json_tool_call but Bedrock