mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-13 08:52:02 +00:00
fix(bedrock): keep the line's structured output path where the picks broke working calls
Backport line fix, not a pick. Every change keeps a request that worked on lts/1.108.x working after the picks in this branch. Models the cost map marks native on every request (Sonnet 4.6, Haiku 4.5, Sonnet 4.5, Mythos 5.1) send the caller's schema the way the line did, on Converse, Invoke and direct Anthropic. #46870 moved some of their schemas to the forced json tool. That turned array roots and typed allOf roots into 400s, gave prose for open objects with thinking on, and broke tool_choice without tools. Opt-in models (Haiku 5.5, Sonnet 5.5, Sonnet 5, Opus 4.8) keep the new native path and its schema checks. bedrock_mantle/ chat has no retry, so Mantle ignores thinking when it picks the path, and opt-in models stay on the json tool there as they did before. The sync Converse stream keeps the line's error handling, so its error classes and router retries stay the same. The retry reads the raw HTTP error there instead. It also sends the caller's headers as they were before the first attempt, so the structured output beta does not leak into it. main has the same routing gaps.
This commit is contained in:
mateo-berri
parent
e2c4dc3d11
commit
040c8ab5bf
16 changed files with 375 additions and 142 deletions
|
|
@ -1681,24 +1681,20 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
json_schema: Mapping[str, object] | None,
|
||||
request_params: Mapping[str, object],
|
||||
) -> bool:
|
||||
open_object_takes_forced_json_tool: Final = (
|
||||
json_schema is not None
|
||||
and schema_has_open_object(json_schema)
|
||||
and not self.is_thinking_requested(request_params)
|
||||
and not self.is_thinking_enabled(dict(request_params))
|
||||
and not request_params.get("tools")
|
||||
and not AnthropicModelInfo.forced_tool_use_unsupported(model)
|
||||
)
|
||||
return self.uses_native_structured_output(model) and not open_object_takes_forced_json_tool
|
||||
return self.uses_native_structured_output(model)
|
||||
|
||||
def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
def native_output_format_schema(
|
||||
self, json_schema: Mapping[str, object], model: str | None = None
|
||||
) -> Mapping[str, object] | None:
|
||||
return self.filter_anthropic_output_schema(dict(self._defs_unpacked(json_schema)))
|
||||
|
||||
def map_response_format_to_anthropic_output_format(self, value: dict | None) -> AnthropicOutputSchema | None:
|
||||
def map_response_format_to_anthropic_output_format(
|
||||
self, value: dict | None, model: str | None = None
|
||||
) -> AnthropicOutputSchema | None:
|
||||
json_schema: Final = self._extract_json_schema_from_response_format(value)
|
||||
if json_schema is None:
|
||||
return None
|
||||
native_schema: Final = self.native_output_format_schema(json_schema)
|
||||
native_schema: Final = self.native_output_format_schema(json_schema, model)
|
||||
if native_schema is None:
|
||||
return None
|
||||
return AnthropicOutputSchema(type="json_schema", schema=dict(native_schema))
|
||||
|
|
@ -1873,7 +1869,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
self._extract_json_schema_from_response_format(value),
|
||||
REQUEST_PARAMS.validate_python(non_default_params),
|
||||
):
|
||||
_output_format = self.map_response_format_to_anthropic_output_format(value)
|
||||
_output_format = self.map_response_format_to_anthropic_output_format(value, model)
|
||||
if _output_format is not None:
|
||||
optional_params["output_format"] = _output_format
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -40,22 +40,14 @@ def make_sync_call(
|
|||
if client is None:
|
||||
client = get_httpx_client() # Create a new client if none provided
|
||||
|
||||
try:
|
||||
response: Final = client.post(
|
||||
api_base,
|
||||
headers=headers,
|
||||
data=data,
|
||||
stream=not fake_stream,
|
||||
logging_obj=logging_obj,
|
||||
timeout=timeout,
|
||||
)
|
||||
except httpx.HTTPStatusError as err:
|
||||
raise BedrockError(
|
||||
status_code=err.response.status_code,
|
||||
message=error_response_text(err.response),
|
||||
headers=err.response.headers,
|
||||
response=err.response,
|
||||
) from err
|
||||
response: Final = client.post(
|
||||
api_base,
|
||||
headers=headers,
|
||||
data=data,
|
||||
stream=not fake_stream,
|
||||
logging_obj=logging_obj,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise BedrockError(
|
||||
|
|
|
|||
|
|
@ -1025,8 +1025,10 @@ class AmazonConverseConfig(BaseConfig):
|
|||
model, self.custom_llm_provider, thinking_requested, model_info
|
||||
):
|
||||
return None
|
||||
if BedrockModelInfo.get_base_model(model).startswith("anthropic"):
|
||||
return bedrock_cleaned_output_schema(model, json_schema, self.custom_llm_provider or "bedrock_converse")
|
||||
if bedrock_native_output_is_opt_in(
|
||||
model, self.custom_llm_provider or "bedrock_converse", model_info
|
||||
) and BedrockModelInfo.get_base_model(model).startswith("anthropic"):
|
||||
return bedrock_cleaned_output_schema(json_schema)
|
||||
return json_schema
|
||||
|
||||
def _output_config_for_anthropic_format(
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
apply_bedrock_invoke_structured_output,
|
||||
bedrock_chat_uses_native_structured_output,
|
||||
bedrock_cleaned_output_schema,
|
||||
bedrock_native_output_is_opt_in,
|
||||
bedrock_supports_tool_search,
|
||||
deployment_model_info,
|
||||
get_anthropic_beta_from_headers,
|
||||
|
|
@ -91,8 +92,12 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
return AnthropicConfig.get_supported_openai_params(self, model)
|
||||
|
||||
def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
return AnthropicConfig.native_output_schema(json_schema)
|
||||
def native_output_format_schema(
|
||||
self, json_schema: Mapping[str, object], model: str | None = None
|
||||
) -> Mapping[str, object] | None:
|
||||
if model is not None and bedrock_native_output_is_opt_in(model, model_info=self._deployment_model_info):
|
||||
return bedrock_cleaned_output_schema(json_schema)
|
||||
return AnthropicConfig.native_output_format_schema(self, json_schema, model)
|
||||
|
||||
def uses_native_structured_output_for_schema(
|
||||
self,
|
||||
|
|
@ -102,7 +107,11 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
) -> bool:
|
||||
return bedrock_chat_uses_native_structured_output(
|
||||
model, self.is_thinking_requested(request_params), model_info=self._deployment_model_info
|
||||
) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None)
|
||||
) and (
|
||||
json_schema is None
|
||||
or not bedrock_native_output_is_opt_in(model, model_info=self._deployment_model_info)
|
||||
or bedrock_cleaned_output_schema(json_schema) is not None
|
||||
)
|
||||
|
||||
def uses_opt_in_native_output(
|
||||
self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
|
||||
|
|
|
|||
|
|
@ -37,6 +37,14 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig):
|
|||
Usage: model="bedrock/mantle/anthropic.claude-mythos-preview"
|
||||
"""
|
||||
|
||||
def uses_native_structured_output_for_schema(
|
||||
self,
|
||||
model: str,
|
||||
json_schema: Mapping[str, object] | None,
|
||||
request_params: Mapping[str, object],
|
||||
) -> bool:
|
||||
return super().uses_native_structured_output_for_schema(model, json_schema, {})
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
from collections.abc import Callable, Coroutine
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
NativeOutputFallback,
|
||||
error_response_text,
|
||||
is_bedrock_native_output_rejection,
|
||||
warn_bedrock_native_output_rejected,
|
||||
)
|
||||
|
|
@ -16,14 +19,24 @@ NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeO
|
|||
NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None]
|
||||
|
||||
|
||||
def _fallback_for(
|
||||
error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str
|
||||
) -> NativeOutputFallback | None:
|
||||
if not is_bedrock_native_output_rejection(error.status_code, error.message):
|
||||
UpstreamError: TypeAlias = BaseLLMException | httpx.HTTPStatusError
|
||||
|
||||
|
||||
def _status_and_text(error: UpstreamError) -> tuple[int, str]:
|
||||
match error:
|
||||
case BaseLLMException():
|
||||
return error.status_code, error.message
|
||||
case httpx.HTTPStatusError():
|
||||
return error.response.status_code, error_response_text(error.response)
|
||||
|
||||
|
||||
def _fallback_for(error: UpstreamError, fallback: NativeOutputFallbackFactory, model: str) -> NativeOutputFallback | None:
|
||||
status_code, error_text = _status_and_text(error)
|
||||
if not is_bedrock_native_output_rejection(status_code, error_text):
|
||||
return None
|
||||
retry: Final = fallback()
|
||||
if retry is not None:
|
||||
warn_bedrock_native_output_rejected(model, error.message)
|
||||
warn_bedrock_native_output_rejected(model, error_text)
|
||||
return retry
|
||||
|
||||
|
||||
|
|
@ -35,7 +48,7 @@ async def _await_with_native_output_fallback(
|
|||
) -> NativeOutputResponse:
|
||||
try:
|
||||
return await first_attempt
|
||||
except BaseLLMException as error:
|
||||
except (BaseLLMException, httpx.HTTPStatusError) as error:
|
||||
retry: Final = _fallback_for(error, fallback, model)
|
||||
if retry is None:
|
||||
raise
|
||||
|
|
@ -52,7 +65,7 @@ def call_with_native_output_fallback(
|
|||
return attempt(None)
|
||||
try:
|
||||
first_attempt: Final = attempt(None)
|
||||
except BaseLLMException as error:
|
||||
except (BaseLLMException, httpx.HTTPStatusError) as error:
|
||||
retry: Final = _fallback_for(error, fallback, model)
|
||||
if retry is None:
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -369,15 +369,10 @@ def json_object_or_none(value: object) -> Mapping[str, object] | None:
|
|||
BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: Final = 64
|
||||
|
||||
|
||||
def bedrock_cleaned_output_schema(
|
||||
model: str, schema: Mapping[str, object], custom_llm_provider: str = "bedrock"
|
||||
) -> Mapping[str, object] | None:
|
||||
def bedrock_cleaned_output_schema(schema: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
if (
|
||||
not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider)
|
||||
and schema_property_count(schema) > BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES
|
||||
):
|
||||
if schema_property_count(schema) > BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES:
|
||||
return None
|
||||
return AnthropicConfig.native_output_schema(schema)
|
||||
|
||||
|
|
@ -388,11 +383,10 @@ def bedrock_native_output_schema(
|
|||
custom_llm_provider: str = "bedrock",
|
||||
model_info: Mapping[str, object] | None = None,
|
||||
) -> Mapping[str, object] | None:
|
||||
native_schema: Final = bedrock_cleaned_output_schema(model, schema, custom_llm_provider)
|
||||
if _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider):
|
||||
return schema if native_schema is None else native_schema
|
||||
return schema
|
||||
if _bedrock_native_output_opt_in(model, custom_llm_provider, model_info):
|
||||
return native_schema
|
||||
return bedrock_cleaned_output_schema(schema)
|
||||
return None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4296,6 +4296,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
|
|||
client=client,
|
||||
)
|
||||
else:
|
||||
headers_before_first_attempt: Final = dict(headers)
|
||||
|
||||
def invoke_attempt(fallback: NativeOutputFallback | None) -> NativeOutputAttemptResult:
|
||||
return base_llm_http_handler.completion(
|
||||
|
|
@ -4309,7 +4310,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
|
|||
litellm_params=litellm_params if fallback is None else dict(fallback.litellm_params),
|
||||
custom_llm_provider="bedrock",
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
headers=headers if fallback is None else dict(headers_before_first_attempt),
|
||||
encoding=_get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.constants import DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS
|
|||
|
||||
_HAIKU: Final = "anthropic.claude-haiku-4-5"
|
||||
_OPUS: Final = "anthropic.claude-opus-5-5"
|
||||
_OPT_IN_NATIVE_OUTPUT: Final = "anthropic.claude-opus-4-8"
|
||||
_SAFEGUARD: Final = "openai.gpt-oss-safeguard-120b"
|
||||
_API_KEY: Final = "synthetic-mantle-bearer"
|
||||
_ACCESS_KEY: Final = "AKIAINTEGRATION000009"
|
||||
|
|
@ -504,6 +505,37 @@ def test_chat_json_schema_response_format_is_sent_as_a_forced_tool_and_returned_
|
|||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", _MESSAGES_PATH)]
|
||||
|
||||
|
||||
def test_chat_json_schema_with_thinking_on_an_opt_in_model_stays_on_the_json_tool(gateway: Gateway) -> None:
|
||||
marker: Final = uuid4().hex
|
||||
thinking: Final[dict[str, JsonValue]] = {"type": "adaptive"}
|
||||
expected: Final = _outbound(
|
||||
_OPT_IN_NATIVE_OUTPUT,
|
||||
[_user_turn(marker)],
|
||||
tools=[{"name": "json_tool_call", "input_schema": _CITY_SCHEMA}],
|
||||
tool_choice={"name": "json_tool_call", "type": "tool"},
|
||||
thinking=thinking,
|
||||
)
|
||||
reply: Final = _tool_reply(marker, _OPT_IN_NATIVE_OUTPUT, "json_tool_call")
|
||||
with wire_server(_bearer_peer(expected, reply)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"bedrock_mantle/{_OPT_IN_NATIVE_OUTPUT}", api_base=wire.url, api_key=_API_KEY
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": _chat_messages(marker),
|
||||
"thinking": thinking,
|
||||
"response_format": {"type": "json_schema", "json_schema": {"name": "weather", "schema": _CITY_SCHEMA}},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
message: Final = object_value(_only_choice(_JSON_OBJECT.validate_json(response.content))["message"])
|
||||
assert json.loads(string_value(message["content"])) == {"city": "Paris"}, response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", _MESSAGES_PATH)]
|
||||
|
||||
|
||||
def _authorization_field(part: str) -> tuple[str, str]:
|
||||
name, _, value = part.partition("=")
|
||||
return name, value
|
||||
|
|
|
|||
|
|
@ -99,16 +99,3 @@ def test_k03_sync_stream_with_normal_frames_delivers_the_text_and_a_stop() -> No
|
|||
assert consumed.finish_reasons[-1] == "stop", consumed
|
||||
assert len(wire.drain()) == 1
|
||||
|
||||
|
||||
def test_k04_sync_stream_open_400_carries_the_bedrock_body_as_text() -> None:
|
||||
body: Final = '{"message":"The request is invalid."}'
|
||||
|
||||
def reject(_: Request) -> Reply:
|
||||
return Reply(status=400, body=body.encode())
|
||||
|
||||
with wire_server(reject) as wire:
|
||||
with pytest.raises(BadRequestError) as raised:
|
||||
_consume_sync_stream(wire)
|
||||
assert raised.value.status_code == 400, raised.value
|
||||
assert f"BedrockException - {body}" in raised.value.message, raised.value.message
|
||||
assert len(wire.drain()) == 1
|
||||
|
|
|
|||
|
|
@ -211,12 +211,18 @@ def _sent_bodies(
|
|||
"output_config": {"effort": "low"},
|
||||
"response_format": _response_format(_SCHEMA),
|
||||
},
|
||||
{
|
||||
"thinking": {"type": "adaptive"},
|
||||
"response_format": _response_format(_SCHEMA),
|
||||
"extra_headers": {"anthropic-beta": "context-1m-2025-08-07"},
|
||||
},
|
||||
],
|
||||
ids=[
|
||||
"reasoning_effort",
|
||||
"constrained_schema",
|
||||
"response_format_and_output_config_format",
|
||||
"adaptive_thinking_with_effort",
|
||||
"caller_beta_header",
|
||||
],
|
||||
)
|
||||
def test_the_retry_sends_what_a_deployment_without_native_output_sends(
|
||||
|
|
|
|||
|
|
@ -1346,15 +1346,15 @@ def test_opus_uses_native_structured_output(model_name):
|
|||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field_schema", "native"),
|
||||
"field_schema",
|
||||
[
|
||||
({"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, True),
|
||||
({"type": "object", "additionalProperties": {"type": "integer"}}, False),
|
||||
({"anyOf": [{"type": "object"}, {"type": "null"}]}, False),
|
||||
{"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
|
||||
{"type": "object", "additionalProperties": {"type": "integer"}},
|
||||
{"anyOf": [{"type": "object"}, {"type": "null"}]},
|
||||
],
|
||||
ids=["closed-object", "dict-field", "nullable-dict-field"],
|
||||
)
|
||||
def test_dict_fields_take_the_json_tool_on_a_native_structured_output_model(field_schema, native):
|
||||
def test_every_field_shape_keeps_native_output_on_a_native_structured_output_model(field_schema):
|
||||
optional_params: Final = AnthropicConfig().map_openai_params(
|
||||
non_default_params={
|
||||
"response_format": {
|
||||
|
|
@ -1370,8 +1370,8 @@ def test_dict_fields_take_the_json_tool_on_a_native_structured_output_model(fiel
|
|||
drop_params=False,
|
||||
)
|
||||
|
||||
assert ("output_format" in optional_params) is native
|
||||
assert ([tool["name"] for tool in optional_params.get("tools", [])] == ["json_tool_call"]) is not native
|
||||
assert "output_format" in optional_params
|
||||
assert "tools" not in optional_params
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -582,13 +582,10 @@ def test_bedrock_native_output_key_applies_only_when_thinking_is_requested(
|
|||
assert ("tools" in result) is not native
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native_key", ["legacy", "bedrock"])
|
||||
@pytest.mark.parametrize("schema", [TREE_SCHEMA, OPEN_OBJECT_SCHEMA], ids=["recursive", "open-object"])
|
||||
def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profile(
|
||||
set_native_structured_output, schema, native_key
|
||||
):
|
||||
def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_an_opt_in_model(set_native_structured_output, schema):
|
||||
model = "us.anthropic.claude-opus-4-8"
|
||||
set_native_structured_output(model, native_key)
|
||||
set_native_structured_output(model, "bedrock")
|
||||
|
||||
result = _invoke_params_for_schema(model, schema)
|
||||
|
||||
|
|
@ -596,6 +593,94 @@ def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profil
|
|||
assert [tool["name"] for tool in result["tools"]] == ["json_tool_call"]
|
||||
|
||||
|
||||
LOOKUP_TOOL: Final = {
|
||||
"type": "function",
|
||||
"function": {"name": "lookup", "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}},
|
||||
}
|
||||
|
||||
ARRAY_ROOT_SCHEMA: Final = {"type": "array", "items": {"type": "object", "additionalProperties": {"type": "string"}}}
|
||||
TYPED_ALL_OF_ROOT_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"allOf": [
|
||||
{"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]},
|
||||
{"type": "object", "properties": {"b": {"type": "string"}}, "required": ["b"]},
|
||||
],
|
||||
}
|
||||
DEFINITIONS_REF_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"properties": {"home": {"$ref": "#/definitions/Address"}},
|
||||
"required": ["home"],
|
||||
"definitions": {"Address": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"schema",
|
||||
[ARRAY_ROOT_SCHEMA, TYPED_ALL_OF_ROOT_SCHEMA, OPEN_OBJECT_SCHEMA, TREE_SCHEMA, DEFINITIONS_REF_SCHEMA],
|
||||
ids=["array-root", "typed-all-of-root", "open-object", "recursive", "definitions-ref"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"request_params",
|
||||
[
|
||||
{},
|
||||
{"thinking": {"type": "adaptive"}},
|
||||
{"reasoning_effort": "low"},
|
||||
{"tool_choice": "none"},
|
||||
{"tool_choice": {"type": "function", "function": {"name": "lookup"}}},
|
||||
],
|
||||
ids=["plain", "thinking", "reasoning-effort", "tool-choice-none", "named-tool-choice-without-tools"],
|
||||
)
|
||||
def test_model_native_on_every_request_sends_every_schema_natively_without_the_json_tool(
|
||||
set_native_structured_output, schema, request_params
|
||||
):
|
||||
model = "us.anthropic.claude-opus-4-8"
|
||||
set_native_structured_output(model, "legacy")
|
||||
|
||||
result = _invoke_params_for_schema(model, schema, request_params)
|
||||
|
||||
assert result["output_format"]["type"] == "json_schema"
|
||||
assert "json_tool_call" not in [tool["name"] for tool in result.get("tools", [])]
|
||||
|
||||
|
||||
def test_definitions_ref_reaches_a_model_native_on_every_request_inlined(set_native_structured_output):
|
||||
model = "us.anthropic.claude-opus-4-8"
|
||||
set_native_structured_output(model, "legacy")
|
||||
|
||||
result = _invoke_params_for_schema(model, DEFINITIONS_REF_SCHEMA, {})
|
||||
|
||||
sent = result["output_format"]["schema"]
|
||||
assert sent["properties"]["home"]["properties"]["city"] == {"type": "string"}
|
||||
assert "$ref" not in json.dumps(sent)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("thinking_params", [{}, {"thinking": {"type": "adaptive"}}], ids=["no-thinking", "thinking"])
|
||||
def test_open_object_schema_stays_native_beside_the_callers_tools_on_a_model_native_on_every_request(
|
||||
set_native_structured_output, thinking_params
|
||||
):
|
||||
model = "us.anthropic.claude-opus-4-8"
|
||||
set_native_structured_output(model, "legacy")
|
||||
|
||||
result = _invoke_params_for_schema(model, OPEN_OBJECT_SCHEMA, {**thinking_params, "tools": [LOOKUP_TOOL]})
|
||||
|
||||
assert "tool_choice" not in result
|
||||
assert [tool["name"] for tool in result["tools"]] == ["lookup"]
|
||||
assert result["output_format"]["schema"]["properties"]["counts"]["additionalProperties"] is False
|
||||
|
||||
|
||||
def test_open_object_schema_keeps_the_json_tool_beside_the_callers_tools_on_an_opt_in_model(
|
||||
set_native_structured_output,
|
||||
):
|
||||
model = "us.anthropic.claude-opus-4-8"
|
||||
set_native_structured_output(model, "bedrock")
|
||||
|
||||
result = _invoke_params_for_schema(
|
||||
model, OPEN_OBJECT_SCHEMA, {"thinking": {"type": "adaptive"}, "tools": [LOOKUP_TOOL]}
|
||||
)
|
||||
|
||||
assert "output_format" not in result
|
||||
assert sorted(tool["name"] for tool in result["tools"]) == ["json_tool_call", "lookup"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native_key", ["legacy", "bedrock"])
|
||||
def test_closed_schema_goes_native_on_either_native_key(set_native_structured_output, native_key):
|
||||
model = "us.anthropic.claude-opus-4-8"
|
||||
|
|
@ -609,10 +694,11 @@ def test_closed_schema_goes_native_on_either_native_key(set_native_structured_ou
|
|||
assert "tools" not in result
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native_key", ["legacy", "bedrock"])
|
||||
def test_shared_defs_reach_native_output_once_instead_of_inlined_per_use(set_native_structured_output, native_key):
|
||||
def test_shared_defs_reach_native_output_once_instead_of_inlined_per_use_on_an_opt_in_model(
|
||||
set_native_structured_output,
|
||||
):
|
||||
model = "us.anthropic.claude-opus-4-8"
|
||||
set_native_structured_output(model, native_key)
|
||||
set_native_structured_output(model, "bedrock")
|
||||
party_ref = {"$ref": "#/$defs/Party"}
|
||||
schema = {
|
||||
"type": "object",
|
||||
|
|
@ -649,12 +735,11 @@ class PuzzleAnswer(BaseModel):
|
|||
relabels: list[Relabel]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native_key", ["legacy", "bedrock"])
|
||||
def test_nested_pydantic_response_format_reaches_native_output_with_refs_that_resolve(
|
||||
set_native_structured_output, native_key
|
||||
def test_nested_pydantic_response_format_reaches_native_output_with_refs_that_resolve_on_an_opt_in_model(
|
||||
set_native_structured_output,
|
||||
):
|
||||
model = "us.anthropic.claude-haiku-5-5"
|
||||
set_native_structured_output(model, native_key)
|
||||
set_native_structured_output(model, "bedrock")
|
||||
config: Final = AmazonAnthropicClaudeConfig()
|
||||
|
||||
result: Final = config.map_openai_params(
|
||||
|
|
|
|||
|
|
@ -346,48 +346,3 @@ def test_converse_profile_override_records_cost_without_changing_response_model(
|
|||
+ CONVERSE_RESPONSE["usage"]["outputTokens"] * prices["output_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
_UPSTREAM_ERROR_BODY: Final = '{"message":"The request is invalid."}'
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize(
|
||||
"model", ["us.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.anthropic.claude-opus-4-8"], ids=["sonnet", "opus"]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("upstream_status", "error_class", "status_code"),
|
||||
[
|
||||
(400, litellm.BadRequestError, 400),
|
||||
(429, litellm.RateLimitError, 429),
|
||||
(500, litellm.ServiceUnavailableError, 503),
|
||||
(503, litellm.ServiceUnavailableError, 503),
|
||||
],
|
||||
)
|
||||
def test_sync_converse_streaming_upstream_error_carries_the_upstream_body_as_text(
|
||||
model: str,
|
||||
upstream_status: int,
|
||||
error_class: type[litellm.BadRequestError | litellm.RateLimitError | litellm.ServiceUnavailableError],
|
||||
status_code: int,
|
||||
) -> None:
|
||||
client: Final = HTTPHandler(
|
||||
client=httpx.Client(
|
||||
transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(upstream_status, content=_UPSTREAM_ERROR_BODY.encode())
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(error_class) as raised:
|
||||
litellm.completion(
|
||||
model=f"bedrock/converse/{model}",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
num_retries=0,
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert raised.value.status_code == status_code
|
||||
assert raised.value.message == f"litellm.{error_class.__name__}: BedrockException - {_UPSTREAM_ERROR_BODY}"
|
||||
|
|
|
|||
|
|
@ -5081,12 +5081,15 @@ def _schema_keys(node: object) -> frozenset[str]:
|
|||
return frozenset()
|
||||
|
||||
|
||||
def _converse_request_for_schema(model: str, schema: Mapping[str, object]) -> RequestObject:
|
||||
def _converse_request_for_schema(
|
||||
model: str, schema: Mapping[str, object], tools: tuple[Mapping[str, object], ...] = ()
|
||||
) -> RequestObject:
|
||||
config: Final = AmazonConverseConfig()
|
||||
optional_params: Final = config.map_openai_params(
|
||||
non_default_params={
|
||||
"response_format": {"type": "json_schema", "json_schema": {"name": "out", "schema": schema}},
|
||||
"thinking": ADAPTIVE_THINKING,
|
||||
**({"tools": list(tools)} if tools else {}),
|
||||
},
|
||||
optional_params={},
|
||||
model=model,
|
||||
|
|
@ -5101,12 +5104,11 @@ def _converse_request_for_schema(model: str, schema: Mapping[str, object]) -> Re
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native_key", ["legacy", "bedrock"])
|
||||
def test_native_output_schema_drops_keywords_bedrock_rejects_and_closes_objects(
|
||||
set_native_structured_output, native_key
|
||||
def test_native_output_schema_drops_keywords_bedrock_rejects_and_closes_objects_on_an_opt_in_model(
|
||||
set_native_structured_output,
|
||||
):
|
||||
model: Final = "us.anthropic.claude-opus-4-8"
|
||||
set_native_structured_output(model, native_key)
|
||||
set_native_structured_output(model, "bedrock")
|
||||
|
||||
request: Final = _converse_request_for_schema(model, CONSTRAINED_PERSON_SCHEMA)
|
||||
|
||||
|
|
@ -5120,7 +5122,6 @@ def test_native_output_schema_drops_keywords_bedrock_rejects_and_closes_objects(
|
|||
assert sent["properties"]["address"]["additionalProperties"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native_key", ["legacy", "bedrock"])
|
||||
@pytest.mark.parametrize(
|
||||
("schema", "kept_keyword"),
|
||||
[
|
||||
|
|
@ -5132,11 +5133,11 @@ def test_native_output_schema_drops_keywords_bedrock_rejects_and_closes_objects(
|
|||
],
|
||||
ids=["recursive", "root-ref", "open-object", "lookaround-pattern", "unlisted-format"],
|
||||
)
|
||||
def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profile(
|
||||
set_native_structured_output, schema, kept_keyword, native_key
|
||||
def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_an_opt_in_model(
|
||||
set_native_structured_output, schema, kept_keyword
|
||||
):
|
||||
model: Final = "us.anthropic.claude-opus-4-8"
|
||||
set_native_structured_output(model, native_key)
|
||||
set_native_structured_output(model, "bedrock")
|
||||
|
||||
request: Final = _converse_request_for_schema(model, schema)
|
||||
|
||||
|
|
@ -5146,6 +5147,102 @@ def test_schema_native_output_cannot_hold_keeps_the_tool_path_on_a_native_profil
|
|||
assert kept_keyword in _schema_keys(tool_spec["inputSchema"]["json"])
|
||||
|
||||
|
||||
LOOKUP_TOOL: Final = {
|
||||
"type": "function",
|
||||
"function": {"name": "lookup", "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}},
|
||||
}
|
||||
|
||||
ARRAY_ROOT_SCHEMA: Final = {"type": "array", "items": {"type": "object", "additionalProperties": {"type": "string"}}}
|
||||
TYPED_ALL_OF_ROOT_SCHEMA: Final = {
|
||||
"type": "object",
|
||||
"allOf": [
|
||||
{"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]},
|
||||
{"type": "object", "properties": {"b": {"type": "string"}}, "required": ["b"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"schema",
|
||||
[
|
||||
ARRAY_ROOT_SCHEMA,
|
||||
TYPED_ALL_OF_ROOT_SCHEMA,
|
||||
OPEN_OBJECT_SCHEMA,
|
||||
CONSTRAINED_PERSON_SCHEMA,
|
||||
LOOKAROUND_PATTERN_SCHEMA,
|
||||
RECURSIVE_TREE_SCHEMA,
|
||||
],
|
||||
ids=["array-root", "typed-all-of-root", "open-object", "constrained", "lookaround-pattern", "recursive"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"request_params",
|
||||
[
|
||||
{},
|
||||
{"thinking": ADAPTIVE_THINKING},
|
||||
{"reasoning_effort": "low"},
|
||||
{"tool_choice": "auto"},
|
||||
{"tool_choice": {"type": "function", "function": {"name": "lookup"}}},
|
||||
],
|
||||
ids=["plain", "thinking", "reasoning-effort", "tool-choice-auto", "named-tool-choice-without-tools"],
|
||||
)
|
||||
def test_model_native_on_every_request_sends_the_callers_schema_natively_with_every_keyword(
|
||||
set_native_structured_output, schema, request_params
|
||||
):
|
||||
model: Final = "us.anthropic.claude-opus-4-8"
|
||||
set_native_structured_output(model, "legacy")
|
||||
config: Final = AmazonConverseConfig()
|
||||
optional_params: Final = config.map_openai_params(
|
||||
non_default_params={
|
||||
"response_format": {"type": "json_schema", "json_schema": {"name": "out", "schema": schema}},
|
||||
**request_params,
|
||||
},
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
request: Final = config._transform_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "Fill in a plausible example."}],
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
sent: Final = json.loads(request["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"])
|
||||
assert _schema_keys(schema) <= _schema_keys(sent)
|
||||
assert "json_tool_call" not in [tool["toolSpec"]["name"] for tool in request.get("toolConfig", {}).get("tools", [])]
|
||||
|
||||
|
||||
def _tool_names(request: RequestObject) -> list[str]:
|
||||
return sorted(tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"])
|
||||
|
||||
|
||||
def test_open_object_schema_stays_native_beside_the_callers_tools_on_a_model_native_on_every_request(
|
||||
set_native_structured_output,
|
||||
):
|
||||
model: Final = "us.anthropic.claude-opus-4-8"
|
||||
set_native_structured_output(model, "legacy")
|
||||
|
||||
request: Final = _converse_request_for_schema(model, OPEN_OBJECT_SCHEMA, (LOOKUP_TOOL,))
|
||||
|
||||
assert "outputConfig" in request
|
||||
assert _tool_names(request) == ["lookup"]
|
||||
assert "toolChoice" not in request["toolConfig"]
|
||||
|
||||
|
||||
def test_open_object_schema_keeps_the_json_tool_beside_the_callers_tools_on_an_opt_in_model(
|
||||
set_native_structured_output,
|
||||
):
|
||||
model: Final = "us.anthropic.claude-opus-4-8"
|
||||
set_native_structured_output(model, "bedrock")
|
||||
|
||||
request: Final = _converse_request_for_schema(model, OPEN_OBJECT_SCHEMA, (LOOKUP_TOOL,))
|
||||
|
||||
assert "outputConfig" not in request
|
||||
assert _tool_names(request) == ["json_tool_call", "lookup"]
|
||||
|
||||
|
||||
def _shared_defs_schema(defs_key: str) -> Mapping[str, object]:
|
||||
party_ref: Final = {"$ref": f"#/{defs_key}/Party"}
|
||||
return {
|
||||
|
|
@ -5195,7 +5292,6 @@ def _optional_groups_schema(count: int, group_size: int = 6) -> Mapping[str, obj
|
|||
return {"type": "object", "properties": groups, "required": list(groups)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native_key", ["legacy", "bedrock"])
|
||||
@pytest.mark.parametrize(
|
||||
("schema_for", "limit"),
|
||||
[
|
||||
|
|
@ -5212,11 +5308,11 @@ def _optional_groups_schema(count: int, group_size: int = 6) -> Mapping[str, obj
|
|||
],
|
||||
ids=["type-list-unions", "any-of-unions", "optional", "optional-grammar-branches"],
|
||||
)
|
||||
def test_schema_past_a_native_output_size_limit_keeps_the_tool_path(
|
||||
set_native_structured_output, native_key, schema_for, limit
|
||||
def test_schema_past_a_native_output_size_limit_keeps_the_tool_path_on_an_opt_in_model(
|
||||
set_native_structured_output, schema_for, limit
|
||||
):
|
||||
model: Final = "us.anthropic.claude-opus-4-8"
|
||||
set_native_structured_output(model, native_key)
|
||||
set_native_structured_output(model, "bedrock")
|
||||
|
||||
at_limit: Final = _converse_request_for_schema(model, schema_for(limit))
|
||||
past_limit: Final = _converse_request_for_schema(model, schema_for(limit + 1))
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.llms.bedrock.chat.mantle.transformation import AmazonMantleConfig
|
|||
from litellm.llms.bedrock.messages.mantle_transformation import (
|
||||
AmazonMantleMessagesConfig,
|
||||
)
|
||||
from litellm.llms.bedrock_mantle.chat.claude_transformation import BedrockMantleClaudeChatConfig
|
||||
|
||||
# AWS names this header for Mantle workspaces on the Anthropic Messages API, checked 2026-10-02:
|
||||
# https://docs.aws.amazon.com/bedrock/latest/userguide/workspaces.html
|
||||
|
|
@ -390,6 +391,62 @@ def test_mantle_messages_validate_environment_without_project_id():
|
|||
assert _MANTLE_WORKSPACE_HEADER not in headers
|
||||
|
||||
|
||||
_MANTLE_CONFIGS = pytest.mark.parametrize(
|
||||
"config", [AmazonMantleConfig(), BedrockMantleClaudeChatConfig()], ids=["bedrock_mantle_route", "mantle_provider"]
|
||||
)
|
||||
_MANTLE_MODEL = "anthropic.claude-opus-4-8"
|
||||
_CLOSED_RESPONSE_FORMAT = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "out",
|
||||
"schema": {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]},
|
||||
},
|
||||
}
|
||||
_OPEN_RESPONSE_FORMAT = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "out",
|
||||
"schema": {"type": "object", "properties": {"counts": {"type": "object", "additionalProperties": True}}},
|
||||
},
|
||||
}
|
||||
_LOOKUP_TOOL = {
|
||||
"type": "function",
|
||||
"function": {"name": "lookup", "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}},
|
||||
}
|
||||
|
||||
|
||||
@_MANTLE_CONFIGS
|
||||
def test_mantle_opt_in_model_keeps_the_json_tool_when_thinking_is_requested(set_native_structured_output, config):
|
||||
set_native_structured_output(_MANTLE_MODEL, "bedrock")
|
||||
|
||||
result = config.map_openai_params(
|
||||
non_default_params={"response_format": _CLOSED_RESPONSE_FORMAT, "thinking": {"type": "adaptive"}},
|
||||
optional_params={},
|
||||
model=_MANTLE_MODEL,
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert "output_format" not in result
|
||||
assert [tool["name"] for tool in result["tools"]] == ["json_tool_call"]
|
||||
|
||||
|
||||
@_MANTLE_CONFIGS
|
||||
def test_mantle_model_native_on_every_request_stays_native_beside_the_callers_tools(
|
||||
set_native_structured_output, config
|
||||
):
|
||||
set_native_structured_output(_MANTLE_MODEL, "legacy")
|
||||
|
||||
result = config.map_openai_params(
|
||||
non_default_params={"response_format": _OPEN_RESPONSE_FORMAT, "tools": [_LOOKUP_TOOL]},
|
||||
optional_params={},
|
||||
model=_MANTLE_MODEL,
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert "output_format" in result
|
||||
assert [tool["name"] for tool in result["tools"]] == ["lookup"]
|
||||
|
||||
|
||||
def test_mantle_completion_sends_workspace_header_and_clean_body():
|
||||
import litellm
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue