scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-31 06:56:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 245

Merge 9a2d61680c into a392a39627

Browse source
This commit is contained in:
ryan-crabbe-berri 2026-10-31 02:46:42 -04:00 • committed by GitHub
commit 2afa605254
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 342 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm/constants.py
View file

@ -1657,6 +1657,7 @@ OPENAI_FINISH_REASONS: Final = [
]
HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = get_env_int("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60) # 1 minute
RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call
JSON_MODE_PREAMBLE_TEXT_KEY: Final = "json_mode_preamble_text"
########################### Logging Callback Constants ###########################
AZURE_STORAGE_MSFT_VERSION: Final = "2019-07-07"

8
litellm/llms/anthropic/chat/transformation.py
View file

@ -22,6 +22,7 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
JSON_MODE_PREAMBLE_TEXT_KEY,
RESPONSE_FORMAT_TOOL_NAME,
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
@ -3059,7 +3060,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
REFUSAL_STOP_FIELDS.validate_python(completion_response)
)
completion_response["stop_reason"] = "stop"
if refusal_fields:
if text_content and text_content.strip():
json_mode_message.provider_specific_fields = {
**refusal_fields,
JSON_MODE_PREAMBLE_TEXT_KEY: text_content,
}
elif refusal_fields:
json_mode_message.provider_specific_fields = dict(refusal_fields)
if thinking_blocks:
json_mode_message.thinking_blocks = thinking_blocks

11
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -21,6 +21,7 @@ from litellm.anthropic_beta_gated_fields import active_beta_values, log_beta_gat
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
from litellm.constants import (
BEDROCK_MIN_THINKING_BUDGET_TOKENS,
JSON_MODE_PREAMBLE_TEXT_KEY,
RESPONSE_FORMAT_TOOL_NAME,
)
from litellm.litellm_core_utils.core_helpers import (
@ -2708,6 +2709,16 @@ class AmazonConverseConfig(BaseConfig):
json_mode_content_str: str | None = tools[0]["function"].get("arguments")
if json_mode_content_str is not None:
json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties(json_mode_content_str)
message_fields: Final = as_str_mapping(chat_completion_message)
existing_content: Final = (message_fields or {}).get("content")
if isinstance(existing_content, str) and existing_content.strip():
existing_provider_fields: Final = _JSON_OBJECT.validate_python(
(message_fields.get("provider_specific_fields") or {}) if message_fields is not None else {}
)
chat_completion_message["provider_specific_fields"] = { # rebind-ok: response is built in place
**existing_provider_fields,
JSON_MODE_PREAMBLE_TEXT_KEY: existing_content,
}
chat_completion_message["content"] = json_mode_content_str
return None

94
tests/e2e/llm_translation/test_bedrock_native_e2e.py
View file

@ -9,6 +9,8 @@ from __future__ import annotations
import pytest
from typing import Final
from openai.types.chat import ChatCompletionMessage
from openai.types.shared_params.response_format_json_schema import ResponseFormatJSONSchema
from pydantic import BaseModel, ConfigDict
from e2e_config import unique_marker
@ -86,6 +88,55 @@ class InvokeResponse(BaseModel):
content: list[ConverseContent]
class WarehouseAnswer(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid", strict=True)
answer: int
class JsonModePreambleFields(BaseModel):
model_config = ConfigDict(frozen=True)
json_mode_preamble_text: str | None = None
class JsonModeAssistantMessage(BaseModel):
model_config = ConfigDict(frozen=True)
content: str | None = None
provider_specific_fields: JsonModePreambleFields | None = None
BEDROCK_JSON_SCHEMA_BACKEND: Final = "bedrock/us.anthropic.claude-opus-5-5"
WAREHOUSE_PROMPT: Final = (
"A warehouse has 3 shelves. Shelf A holds 17 boxes of 24 items, shelf B holds 9 boxes of 36 items minus 13 "
"damaged items, and shelf C holds twice as many items as shelf B. How many usable items are there in total? "
"Work through it carefully, then answer."
)
WAREHOUSE_RESPONSE_FORMAT: Final[ResponseFormatJSONSchema] = {
"type": "json_schema",
"json_schema": {
"name": "warehouse_answer",
"strict": True,
"schema": {
"type": "object",
"properties": {"answer": {"type": "integer"}},
"required": ["answer"],
"additionalProperties": False,
},
},
}
def _warehouse_message_has_preamble(message: ChatCompletionMessage) -> bool:
structured_message: Final = JsonModeAssistantMessage.model_validate(message.model_dump())
assert structured_message.content is not None
WarehouseAnswer.model_validate_json(structured_message.content)
provider_fields: Final = structured_message.provider_specific_fields
preamble: Final = provider_fields.json_mode_preamble_text if provider_fields is not None else None
return preamble is not None and bool(preamble.strip())
def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]:
model = f"e2e-bedrock-native-{unique_marker()}"
model_id = proxy.create_model(
@ -144,6 +195,49 @@ class TestBedrockNative:
assert parsed.events
assert all(event.name and event.date and event.participants for event in parsed.events)
@pytest.mark.provider_live
@pytest.mark.covers("llm.chat_completions.bedrock_converse.structured_output.preamble.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.CHAT_COMPLETIONS,
providers=(Provider.BEDROCK,),
models=(BEDROCK_JSON_SCHEMA_BACKEND,),
capabilities=(Capability.RESPONSE_SCHEMA,),
mode=Mode.NONSTREAM,
)
)
def test_bedrock_json_schema_response_exposes_text_before_json_tool_call(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = f"e2e-bedrock-json-mode-preamble-{unique_marker()}"
model_id: Final = proxy.create_model(
model,
LiteLLMParamsBody(
model=BEDROCK_JSON_SCHEMA_BACKEND,
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
aws_region_name="us-east-1",
),
)
resources.defer(lambda: proxy.delete_model(model_id))
key: Final = resources.key()
assert any(
_warehouse_message_has_preamble(
sdk.openai(key)
.chat.completions.create(
model=model,
messages=[{"role": "user", "content": WAREHOUSE_PROMPT}],
response_format=WAREHOUSE_RESPONSE_FORMAT,
max_tokens=1500,
)
.choices[0]
.message
)
for _ in range(3)
), "No model-generated preamble appeared in three Bedrock calls"
@pytest.mark.covers("llm.bedrock_native.bedrock_converse.basic.nonstream.works")
@meta(
Subject(

63
tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -9761,6 +9761,69 @@ def test_json_mode_keeps_the_thinking_that_preceded_the_json_tool_call():
assert [block["signature"] for block in message.thinking_blocks] == [thinking_block["signature"]]
def test_json_mode_preserves_nonblank_text_before_json_tool_call():
preamble: Final = "Let me calculate the usable items. "
body: Final = _json_mode_message_body("tool_use", None)
result: Final = _transform_json_mode_body(
{**body, "content": [{"type": "text", "text": preamble}, *body["content"]]}
)
message: Final = result.choices[0].message
assert message.content == '{"answer": "Hello"}'
assert message.provider_specific_fields["json_mode_preamble_text"] == preamble
def test_json_mode_preamble_preserves_refusal_provider_fields():
preamble: Final = "Let me explain the result. "
stop_details: Final = ANTHROPIC_DOCS_REFUSAL_MESSAGE["stop_details"]
body: Final = _json_mode_message_body("refusal", stop_details)
result: Final = _transform_json_mode_body(
{**body, "content": [{"type": "text", "text": preamble}, *body["content"]]}
)
message: Final = result.choices[0].message
assert message.content == '{"answer": "Hello"}'
assert message.provider_specific_fields == {
"stop_reason": "refusal",
"stop_details": stop_details,
"json_mode_preamble_text": preamble,
}
def test_json_mode_text_without_tool_call_is_unchanged():
body: Final = _json_mode_message_body("tool_use", None)
result: Final = _transform_json_mode_body(
{**body, "content": [{"type": "text", "text": "A plain response."}]}
)
message: Final = result.choices[0].message
assert message.content == "A plain response."
assert message.tool_calls is None
assert "json_mode_preamble_text" not in (message.provider_specific_fields or {})
def test_json_mode_with_a_real_tool_keeps_mixed_response_behavior():
preamble: Final = "I will look up the details. "
body: Final = _json_mode_message_body("tool_use", None)
user_tool_call: Final = {
"type": "tool_use",
"id": "toolu_user",
"name": "get_weather",
"input": {"location": "San Francisco"},
}
result: Final = _transform_json_mode_body(
{**body, "content": [{"type": "text", "text": preamble}, *body["content"], user_tool_call]}
)
message: Final = result.choices[0].message
assert message.content == f'{preamble}{{"answer": "Hello"}}'
assert message.tool_calls is not None
assert len(message.tool_calls) == 1
assert message.tool_calls[0].function.name == "get_weather"
assert message.tool_calls[0].function.arguments == '{"location": "San Francisco"}'
assert "json_mode_preamble_text" not in (message.provider_specific_fields or {})
def test_non_dict_upstream_body_still_transforms_without_refusal_fields() -> None:
result = AnthropicConfig().transform_parsed_response(
completion_response=MagicMock(),

168
tests/unit/llms/bedrock/chat/test_converse_transformation.py
View file

@ -4,13 +4,14 @@ import json
import logging
import os
import re
from collections.abc import Iterator, Mapping
from typing import Final
from collections.abc import Iterable, Iterator, Mapping
from typing import Final, cast
from unittest.mock import MagicMock, Mock, patch
import httpx
import pytest
import respx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -31,6 +32,7 @@ from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTI
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.types.llms.bedrock import ContentBlock, ConverseTokenUsageBlock, RequestObject
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponseStream
def test_transform_usage():
@ -6834,6 +6836,168 @@ def test_transform_response_finish_reason_stop_when_json_mode_filters_all_tools(
assert result.choices[0].finish_reason == "stop"
def _transform_json_mode_content_blocks(content: list[dict[str, object]]) -> ModelResponse:
response_json: Final = {
"metrics": {"latencyMs": 100},
"output": {"message": {"role": "assistant", "content": content}},
"stopReason": "tool_use",
"usage": {
"inputTokens": 50,
"outputTokens": 20,
"totalTokens": 70,
"cacheReadInputTokenCount": 0,
"cacheReadInputTokens": 0,
"cacheWriteInputTokenCount": 0,
"cacheWriteInputTokens": 0,
},
}
class MockResponse:
def json(self) -> dict[str, object]:
return response_json
@property
def text(self) -> str:
return json.dumps(response_json)
return AmazonConverseConfig()._transform_response(
model="bedrock/us.anthropic.claude-haiku-5-5",
response=MockResponse(),
model_response=ModelResponse(),
stream=False,
logging_obj=None,
optional_params={"json_mode": True},
api_key=None,
data=None,
messages=[],
encoding=None,
)
def test_bedrock_json_mode_preserves_nonblank_text_before_json_tool_call():
preamble: Final = "Let me calculate the usable items. "
result: Final = _transform_json_mode_content_blocks(
[
{"text": preamble},
{
"toolUse": {
"toolUseId": "tooluse_json_001",
"name": "json_tool_call",
"input": {"answer": 42},
}
},
]
)
message: Final = result.choices[0].message
assert message.content == '{"answer": 42}'
assert message.provider_specific_fields["json_mode_preamble_text"] == preamble
def test_bedrock_json_mode_fake_stream_preserves_preamble_in_chunks() -> None:
preamble: Final = "Let me calculate the usable items. "
response_json: Final = {
"output": {
"message": {
"role": "assistant",
"content": [
{"text": preamble},
{
"toolUse": {
"toolUseId": "tooluse_json_001",
"name": "json_tool_call",
"input": {"answer": 42},
}
},
],
}
},
"stopReason": "tool_use",
"usage": {"inputTokens": 50, "outputTokens": 20, "totalTokens": 70},
}
def respond(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json=response_json, request=request)
model: Final = "bedrock/us.anthropic.claude-opus-5-5"
client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
stream: Final = completion(
model=model,
messages=[{"role": "user", "content": "How many usable items are there?"}],
response_format={
"type": "json_schema",
"json_schema": {
"name": "answer",
"schema": {
"type": "object",
"properties": {"answer": {"type": "integer"}},
"required": ["answer"],
"additionalProperties": False,
},
},
},
stream=True,
aws_access_key_id="fake",
aws_secret_access_key="fake",
aws_region_name="us-east-1",
client=client,
)
chunks: Final = list(cast(Iterable[ModelResponseStream], stream))
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) == '{"answer": 42}'
provider_fields_adapter: Final = TypeAdapter(Mapping[str, object])
provider_fields: Final = tuple(
provider_fields_adapter.validate_python(chunk.choices[0].delta.provider_specific_fields or {})
for chunk in chunks
if chunk.choices
)
assert any(fields.get("json_mode_preamble_text") == preamble for fields in provider_fields)
def test_bedrock_json_mode_omits_whitespace_only_preamble():
result: Final = _transform_json_mode_content_blocks(
[
{"text": " \n\t"},
{
"toolUse": {
"toolUseId": "tooluse_json_001",
"name": "json_tool_call",
"input": {"answer": 42},
}
},
]
)
message: Final = result.choices[0].message
assert message.content == '{"answer": 42}'
assert "json_mode_preamble_text" not in (message.provider_specific_fields or {})
def test_bedrock_json_mode_preamble_preserves_reasoning_content_fields():
reasoning_content: Final = {
"reasoningText": {"text": "The answer is 42.", "signature": "signature-1"}
}
preamble: Final = "Here is the result. "
result: Final = _transform_json_mode_content_blocks(
[
{"reasoningContent": reasoning_content},
{"text": preamble},
{
"toolUse": {
"toolUseId": "tooluse_json_001",
"name": "json_tool_call",
"input": {"answer": 42},
}
},
]
)
message: Final = result.choices[0].message
assert message.content == '{"answer": 42}'
assert message.provider_specific_fields["json_mode_preamble_text"] == preamble
assert message.provider_specific_fields["reasoningContentBlocks"] == [reasoning_content]
def test_transform_response_json_mode_truncated_tool_call_keeps_length_finish_reason():
"""
When json_mode filters out the synthetic json_tool_call but Bedrock