mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
fix(bedrock): retry without strict when Bedrock rejects strict tool schemas under drop_params (#48733)
* fix(bedrock): retry without strict when Bedrock rejects strict tool schemas under drop_params * test(bedrock): cover the strict tools 400 retry under drop_params * fix(bedrock): anchor the strict tools rejection match and strip toolSpec strict * test(bedrock): return captured requests from the strict tools test helper * fix(bedrock): retry without strict only when a tool sent strict to Bedrock A strict-limit 400 on a request whose tools carry strict: false, or on a model litellm never sends strict to, was sent again with an identical body. The fallback now takes the model and returns nothing unless a tool puts strict on the wire. Also types the fallback test parameter and records the mock transport's requests on the recorder instead of a mutated list * fix(bedrock): decide the strict retry on the profile's base model and retry grammar timeouts An application inference profile with base_model sends strict on the wire, so the retry check now resolves the capability model the way the request transformation does. Bedrock's "Grammar compilation timed out" 400 is a strict-grammar rejection too, so it takes the same single retry under drop_params. * test(bedrock): annotate the strict-tools retry test helper's values as Final --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
parent
e2d78b69cd
commit
a0c490ea54
6 changed files with 412 additions and 12 deletions
|
|
@ -1,11 +1,14 @@
|
|||
from collections.abc import Callable, Coroutine
|
||||
from functools import partial
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
NativeOutputFallback,
|
||||
is_bedrock_native_output_rejection,
|
||||
is_bedrock_strict_tools_rejection,
|
||||
warn_bedrock_native_output_rejected,
|
||||
warn_bedrock_strict_tools_dropped,
|
||||
)
|
||||
from litellm.types.utils import ModelResponse
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
|
|
@ -14,12 +17,13 @@ NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper
|
|||
NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse]
|
||||
NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult]
|
||||
NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None]
|
||||
_RetryFor: TypeAlias = Callable[[BaseLLMException], NativeOutputFallback | None]
|
||||
|
||||
|
||||
def _fallback_for(
|
||||
error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str
|
||||
def _native_output_retry(
|
||||
error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
|
||||
) -> NativeOutputFallback | None:
|
||||
if not is_bedrock_native_output_rejection(error.status_code, error.message):
|
||||
if fallback is None or not is_bedrock_native_output_rejection(error.status_code, error.message):
|
||||
return None
|
||||
retry: Final = fallback()
|
||||
if retry is not None:
|
||||
|
|
@ -27,16 +31,38 @@ def _fallback_for(
|
|||
return retry
|
||||
|
||||
|
||||
def _strict_tools_retry(
|
||||
error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
|
||||
) -> NativeOutputFallback | None:
|
||||
if fallback is None or not is_bedrock_strict_tools_rejection(error.status_code, error.message):
|
||||
return None
|
||||
retry: Final = fallback()
|
||||
if retry is not None:
|
||||
warn_bedrock_strict_tools_dropped(model)
|
||||
return retry
|
||||
|
||||
|
||||
def _fallback_for(
|
||||
error: BaseLLMException,
|
||||
fallback: NativeOutputFallbackFactory | None,
|
||||
strict_tools_fallback: NativeOutputFallbackFactory | None,
|
||||
model: str,
|
||||
) -> NativeOutputFallback | None:
|
||||
native_output_retry: Final = _native_output_retry(error, fallback, model)
|
||||
if native_output_retry is not None:
|
||||
return native_output_retry
|
||||
return _strict_tools_retry(error, strict_tools_fallback, model)
|
||||
|
||||
|
||||
async def _await_with_native_output_fallback(
|
||||
first_attempt: Coroutine[object, object, NativeOutputResponse],
|
||||
attempt: NativeOutputAttempt,
|
||||
fallback: NativeOutputFallbackFactory,
|
||||
model: str,
|
||||
retry_for: _RetryFor,
|
||||
) -> NativeOutputResponse:
|
||||
try:
|
||||
return await first_attempt
|
||||
except BaseLLMException as error:
|
||||
retry: Final = _fallback_for(error, fallback, model)
|
||||
retry: Final = retry_for(error)
|
||||
if retry is None:
|
||||
raise
|
||||
second_attempt: Final = attempt(retry)
|
||||
|
|
@ -47,16 +73,20 @@ def call_with_native_output_fallback(
|
|||
attempt: NativeOutputAttempt,
|
||||
fallback: NativeOutputFallbackFactory | None,
|
||||
model: str,
|
||||
strict_tools_fallback: NativeOutputFallbackFactory | None = None,
|
||||
) -> NativeOutputAttemptResult:
|
||||
if fallback is None:
|
||||
if fallback is None and strict_tools_fallback is None:
|
||||
return attempt(None)
|
||||
retry_for: Final = partial(
|
||||
_fallback_for, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model
|
||||
)
|
||||
try:
|
||||
first_attempt: Final = attempt(None)
|
||||
except BaseLLMException as error:
|
||||
retry: Final = _fallback_for(error, fallback, model)
|
||||
retry: Final = retry_for(error)
|
||||
if retry is None:
|
||||
raise
|
||||
return attempt(retry)
|
||||
if isinstance(first_attempt, Coroutine):
|
||||
return _await_with_native_output_fallback(first_attempt, attempt, fallback, model)
|
||||
return _await_with_native_output_fallback(first_attempt, attempt, retry_for)
|
||||
return first_attempt
|
||||
|
|
|
|||
|
|
@ -362,6 +362,60 @@ class NativeOutputFallback:
|
|||
litellm_params: Mapping[str, object]
|
||||
|
||||
|
||||
_BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile(
|
||||
r"Too many strict tools \(\d+\)"
|
||||
r"|Schemas contains too many (?:optional parameters|parameters with union types)"
|
||||
r"|[Cc]ompiled grammar (?:is too large|size)"
|
||||
r"|tools\.\d+\.custom(?:\.strict)?: "
|
||||
r"|Grammar compilation timed out"
|
||||
)
|
||||
_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
_TOOL_SPEC_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool:
|
||||
return status_code == 400 and _BEDROCK_STRICT_TOOLS_REJECTION.search(error_text) is not None
|
||||
|
||||
|
||||
def warn_bedrock_strict_tools_dropped(model: str) -> None:
|
||||
verbose_logger.warning(
|
||||
"Bedrock rejected the strict tool schemas for model=%s, so litellm sent the request again without `strict` "
|
||||
"because drop_params is set.",
|
||||
model,
|
||||
)
|
||||
|
||||
|
||||
def _without_strict(tool: Mapping[str, object]) -> Mapping[str, object]:
|
||||
spec_key: Final = next((key for key in ("function", "toolSpec") if key in tool), None)
|
||||
if spec_key is None:
|
||||
return tool
|
||||
spec: Final = _TOOL_SPEC_ADAPTER.validate_python(tool[spec_key])
|
||||
return {**tool, spec_key: {key: value for key, value in spec.items() if key != "strict"}}
|
||||
|
||||
|
||||
def _sends_strict(tool: Mapping[str, object], supports_strict_tools: bool) -> bool:
|
||||
if "toolSpec" in tool:
|
||||
return "strict" in _TOOL_SPEC_ADAPTER.validate_python(tool["toolSpec"])
|
||||
if "function" not in tool:
|
||||
return False
|
||||
return supports_strict_tools and bool(_TOOL_SPEC_ADAPTER.validate_python(tool["function"]).get("strict"))
|
||||
|
||||
|
||||
def bedrock_strict_tools_fallback(
|
||||
model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
|
||||
) -> NativeOutputFallback | None:
|
||||
tools: Final = _TOOLS_ADAPTER.validate_python(optional_params.get("tools", ()))
|
||||
supports_strict_tools: Final = bedrock_converse_supports_strict_tools(
|
||||
bedrock_capability_model(model, litellm_params.get("base_model"))
|
||||
)
|
||||
if not any(_sends_strict(tool, supports_strict_tools) for tool in tools):
|
||||
return None
|
||||
return NativeOutputFallback(
|
||||
optional_params={**optional_params, "tools": [_without_strict(tool) for tool in tools]},
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
|
||||
def requests_output_config_format(optional_params: Mapping[str, object]) -> bool:
|
||||
match optional_params:
|
||||
case {"output_config": {"format": object()}}:
|
||||
|
|
|
|||
|
|
@ -123,6 +123,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
BedrockModelInfo,
|
||||
NativeOutputFallback,
|
||||
bedrock_route_for_request,
|
||||
bedrock_strict_tools_fallback,
|
||||
deployment_model_info,
|
||||
without_bedrock_native_output_opt_in,
|
||||
without_bedrock_route_prefix,
|
||||
|
|
@ -206,7 +207,12 @@ from litellm.utils import (
|
|||
|
||||
from ._logging import verbose_logger
|
||||
from .caching.caching import disable_cache, enable_cache, update_cache
|
||||
from .litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params, safe_deep_copy
|
||||
from .litellm_core_utils.core_helpers import (
|
||||
as_str_mapping,
|
||||
drops_unsupported_params,
|
||||
normalize_drop_params,
|
||||
safe_deep_copy,
|
||||
)
|
||||
from .litellm_core_utils.fallback_utils import (
|
||||
async_completion_with_fallbacks,
|
||||
completion_with_fallbacks,
|
||||
|
|
@ -4378,6 +4384,11 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
|
|||
else None
|
||||
),
|
||||
converse_model,
|
||||
strict_tools_fallback=(
|
||||
partial(bedrock_strict_tools_fallback, converse_model, converse_params, dict(litellm_params))
|
||||
if drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
else None
|
||||
),
|
||||
)
|
||||
elif bedrock_route == "converse_like":
|
||||
model = model.replace("converse_like/", "")
|
||||
|
|
|
|||
|
|
@ -121,3 +121,105 @@ def test_a_rejection_logs_the_retry_and_advises_opting_out_only_when_native_outp
|
|||
|
||||
assert _MODEL in caplog.text
|
||||
assert ("supports_bedrock_native_structured_output: false" in caplog.text) is advises_opting_out
|
||||
|
||||
|
||||
_STRICT_FALLBACK: Final = NativeOutputFallback(optional_params={"tools": "without_strict"}, litellm_params={})
|
||||
# Bedrock's 400s for strict tools, seen live on us.anthropic.claude-sonnet-4-6 from 2026-10-20 to 2026-10-28 and quoted
|
||||
# in issue #34388, each answered 200 once the request dropped strict
|
||||
_STRICT_TOOLS_REJECTIONS: Final = tuple(
|
||||
BedrockError(status_code=400, message=f'{{"message":"The model returned the following errors: {text}"}}')
|
||||
for text in (
|
||||
"Too many strict tools (21). The maximum number of strict tools supported is 20.",
|
||||
"Schemas contains too many optional parameters (27), which would make grammar compilation inefficient.",
|
||||
"Schemas contains too many parameters with union types (18 parameters with type arrays or anyOf). "
|
||||
"This causes exponential compilation cost.",
|
||||
"The compiled grammar is too large, which would cause performance issues.",
|
||||
"tools.0.custom: Invalid schema: Schema type is missing for schema: {'description': 'tag'}",
|
||||
"tools.3.custom: For 'integer' type, property 'minimum' is not supported",
|
||||
"Compiled grammar size (410.4MB) exceeds maximum allowed size (300MB)",
|
||||
"tools.1.custom.strict: Extra inputs are not permitted",
|
||||
"Grammar compilation timed out.",
|
||||
)
|
||||
)
|
||||
_TOO_MANY_STRICT_TOOLS: Final = _STRICT_TOOLS_REJECTIONS[0]
|
||||
# Bedrock's 400s for requests with a strict tool that fail for other reasons, seen live on
|
||||
# us.anthropic.claude-sonnet-4-6 on 2026-10-21, which echo tool names and tool call ids
|
||||
_OTHER_400S_WITH_A_STRICT_TOOL: Final = tuple(
|
||||
BedrockError(status_code=400, message=f'{{"message":"{text}"}}')
|
||||
for text in (
|
||||
"The tool check_grammar is already defined at toolConfig.tools.1.",
|
||||
"Expected toolResult blocks at messages.0.content for the following Ids: call_grammar_9",
|
||||
"tool_use ids were found without tool_result blocks immediately after: call_grammar_1",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _call_with_strict_tools_fallback(
|
||||
bedrock: ScriptedBedrock, fallback: NativeOutputFallback | None, strict_tools_fallback: NativeOutputFallback | None
|
||||
) -> object:
|
||||
return _resolve(
|
||||
call_with_native_output_fallback(
|
||||
bedrock,
|
||||
None if fallback is None else lambda: fallback,
|
||||
_MODEL,
|
||||
strict_tools_fallback=None if strict_tools_fallback is None else lambda: strict_tools_fallback,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@_MODES
|
||||
@pytest.mark.parametrize("rejection", _STRICT_TOOLS_REJECTIONS, ids=lambda error: error.message[60:100])
|
||||
def test_a_strict_tools_rejection_is_sent_again_once_with_the_strict_tools_fallback(
|
||||
asynchronous: bool, rejection: BedrockError
|
||||
) -> None:
|
||||
bedrock: Final = ScriptedBedrock(errors=(rejection, None), asynchronous=asynchronous)
|
||||
|
||||
response: Final = _call_with_strict_tools_fallback(bedrock, None, _STRICT_FALLBACK)
|
||||
|
||||
assert isinstance(response, ModelResponse) and response.id == "answer-2"
|
||||
assert bedrock.sent == (None, _STRICT_FALLBACK)
|
||||
|
||||
|
||||
@_MODES
|
||||
@pytest.mark.parametrize(
|
||||
("error", "strict_tools_fallback"),
|
||||
[
|
||||
(_OTHER_400, _STRICT_FALLBACK),
|
||||
*((error, _STRICT_FALLBACK) for error in _OTHER_400S_WITH_A_STRICT_TOOL),
|
||||
(BedrockError(status_code=500, message=_TOO_MANY_STRICT_TOOLS.message), _STRICT_FALLBACK),
|
||||
(_TOO_MANY_STRICT_TOOLS, None),
|
||||
],
|
||||
ids=[
|
||||
"other_400",
|
||||
"duplicate_tool_name",
|
||||
"missing_tool_result",
|
||||
"orphan_tool_use",
|
||||
"server_error",
|
||||
"no_strict_tools_fallback",
|
||||
],
|
||||
)
|
||||
def test_no_second_request_without_both_a_strict_tools_rejection_and_a_strict_tools_fallback(
|
||||
asynchronous: bool, error: BedrockError, strict_tools_fallback: NativeOutputFallback | None
|
||||
) -> None:
|
||||
bedrock: Final = ScriptedBedrock(errors=(error,), asynchronous=asynchronous)
|
||||
|
||||
with pytest.raises(BedrockError) as raised:
|
||||
_call_with_strict_tools_fallback(bedrock, None, strict_tools_fallback)
|
||||
|
||||
assert raised.value is error
|
||||
assert bedrock.sent == (None,)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("error", "retried_with"),
|
||||
[(_REJECTION, _FALLBACK), (_TOO_MANY_STRICT_TOOLS, _STRICT_FALLBACK)],
|
||||
ids=["native_output_rejection", "strict_tools_rejection"],
|
||||
)
|
||||
def test_each_rejection_is_sent_again_with_its_own_fallback_when_both_are_set(
|
||||
error: BedrockError, retried_with: NativeOutputFallback
|
||||
) -> None:
|
||||
bedrock: Final = ScriptedBedrock(errors=(error, None), asynchronous=False)
|
||||
|
||||
_call_with_strict_tools_fallback(bedrock, _FALLBACK, _STRICT_FALLBACK)
|
||||
|
||||
assert bedrock.sent == (None, retried_with)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,9 @@ from typing import Final
|
|||
from litellm.llms.bedrock.common_utils import(
|
||||
AmazonBedrockGlobalConfig,
|
||||
BedrockModelInfo,
|
||||
NativeOutputFallback,
|
||||
bedrock_native_output_is_opt_in,
|
||||
bedrock_strict_tools_fallback,
|
||||
deployment_model_info,
|
||||
extract_model_name_from_bedrock_arn,
|
||||
get_bedrock_base_model,
|
||||
|
|
@ -2150,3 +2152,80 @@ def test_bedrock_response_stream_shape_is_unavailable_when_botocore_describes_th
|
|||
assert shape is None
|
||||
(logged,) = warning.call_args_list
|
||||
assert "only-in-the-description" not in logged.args[0] % logged.args[1:]
|
||||
|
||||
|
||||
def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_else():
|
||||
strict_tool: Final = {
|
||||
"type": "function",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
"function": {"name": "lookup", "description": "Look up", "strict": True, "parameters": {"type": "object"}},
|
||||
}
|
||||
plain_tool: Final = {"type": "function", "function": {"name": "plain", "parameters": {"type": "object"}}}
|
||||
anthropic_tool: Final = {"name": "native", "input_schema": {"type": "object"}}
|
||||
bedrock_tool: Final = {"toolSpec": {"name": "spec", "strict": True, "inputSchema": {"json": {"type": "object"}}}}
|
||||
litellm_params: Final = {"drop_params": True}
|
||||
|
||||
fallback: Final = bedrock_strict_tools_fallback(
|
||||
"us.anthropic.claude-sonnet-4-6",
|
||||
{"tools": [strict_tool, plain_tool, anthropic_tool, bedrock_tool], "maxTokens": 16, "stream": True},
|
||||
litellm_params,
|
||||
)
|
||||
|
||||
assert fallback == NativeOutputFallback(
|
||||
optional_params={
|
||||
"tools": [
|
||||
{**strict_tool, "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}},
|
||||
plain_tool,
|
||||
anthropic_tool,
|
||||
{"toolSpec": {"name": "spec", "inputSchema": {"json": {"type": "object"}}}},
|
||||
],
|
||||
"maxTokens": 16,
|
||||
"stream": True,
|
||||
},
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
assert strict_tool["function"]["strict"] is True
|
||||
assert bedrock_tool["toolSpec"]["strict"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"optional_params",
|
||||
[
|
||||
{"maxTokens": 16},
|
||||
{"tools": [{"type": "function", "function": {"name": "plain", "parameters": {}}}]},
|
||||
{"tools": [{"toolSpec": {"name": "spec", "inputSchema": {"json": {}}}}]},
|
||||
],
|
||||
ids=["no_tools", "no_strict_tool", "no_strict_tool_spec"],
|
||||
)
|
||||
def test_bedrock_strict_tools_fallback_is_none_without_a_strict_tool(optional_params: dict[str, object]) -> None:
|
||||
assert bedrock_strict_tools_fallback("us.anthropic.claude-sonnet-4-6", optional_params, {}) is None
|
||||
|
||||
|
||||
_CLAUDE_PROFILE_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "litellm_params"),
|
||||
[
|
||||
("us.anthropic.claude-sonnet-4-6", {}),
|
||||
("us.amazon.nova-pro-v1:0", {}),
|
||||
(_CLAUDE_PROFILE_ARN, {"base_model": "bedrock/us.anthropic.claude-sonnet-4-6"}),
|
||||
(_CLAUDE_PROFILE_ARN, {}),
|
||||
],
|
||||
ids=["claude", "nova", "claude_profile_with_base_model", "profile_without_base_model"],
|
||||
)
|
||||
@pytest.mark.parametrize("strict", [True, False])
|
||||
def test_bedrock_strict_tools_fallback_retries_only_when_the_request_sent_strict(
|
||||
model: str, litellm_params: dict[str, object], strict: bool
|
||||
) -> None:
|
||||
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
|
||||
|
||||
tools: Final = [{"type": "function", "function": {"name": "lookup", "strict": strict, "parameters": {}}}]
|
||||
request: Final = AmazonConverseConfig().transform_request(
|
||||
model, [{"role": "user", "content": "hi"}], {"tools": tools}, {**litellm_params}, {}
|
||||
)
|
||||
sent_strict: Final = any("strict" in tool["toolSpec"] for tool in request["toolConfig"]["tools"])
|
||||
|
||||
fallback: Final = bedrock_strict_tools_fallback(model, {"tools": tools}, litellm_params)
|
||||
|
||||
assert (fallback is not None) == sent_strict
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from dataclasses import dataclass
|
|||
from datetime import datetime
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -44,12 +44,13 @@ from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kw
|
|||
from litellm.litellm_core_utils.get_litellm_params import stored_control_options
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.types.litellm_params import ControlOptions
|
||||
from litellm.types.llms.openai import AllMessageValues, HttpxBinaryResponseContent
|
||||
from litellm.types.prompts.init_prompts import PromptSpec
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StandardCallbackDynamicParams, StreamingChoices, Usage
|
||||
from tests.unit.llms.bedrock.slow_upstream import (
|
||||
CONVERSE_ANSWER,
|
||||
STREAM_TIMEOUT_SECONDS,
|
||||
UPSTREAM_FIRST_BYTE_SECONDS,
|
||||
slow_upstream_sync_client,
|
||||
|
|
@ -9950,3 +9951,126 @@ async def test_converse_fallback_prices_the_current_attempt(
|
|||
else:
|
||||
client.close()
|
||||
configuration.reset_rust_configuration()
|
||||
|
||||
|
||||
_LOOKUP_FUNCTION: Final = {
|
||||
"name": "lookup_record",
|
||||
"description": "Look up a record",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
"required": ["query"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
}
|
||||
_STRICT_LOOKUP_TOOL: Final = {"type": "function", "function": {**_LOOKUP_FUNCTION, "strict": True}}
|
||||
# Bedrock's 400 for a request past Claude's strict tool limits, seen live on us.anthropic.claude-sonnet-4-6 on 2026-10-20
|
||||
_TOO_MANY_STRICT_TOOLS: Final = (
|
||||
"The model returned the following errors: Too many strict tools (21). The maximum number of strict tools "
|
||||
"supported is 20. Try reducing the number of tools marked as strict."
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _StrictToolsLimitRejectionFirst:
|
||||
stream: Literal["off", "fake", "real"]
|
||||
sent: tuple[tuple[str, dict[str, object]], ...] = ()
|
||||
|
||||
def __call__(self, request: httpx.Request) -> httpx.Response:
|
||||
self.sent = (*self.sent, (request.url.path.rsplit("/", 1)[-1], json.loads(request.content)))
|
||||
if len(self.sent) == 1:
|
||||
return httpx.Response(400, json={"message": _TOO_MANY_STRICT_TOOLS})
|
||||
return httpx.Response(200, content=b"" if self.stream == "real" else CONVERSE_ANSWER)
|
||||
|
||||
|
||||
async def _converse_with_a_strict_tools_limit_rejection_first(
|
||||
asynchronous: bool,
|
||||
stream: Literal["off", "fake", "real"],
|
||||
drop_params: bool | None,
|
||||
tool: Mapping[str, object] = _STRICT_LOOKUP_TOOL,
|
||||
) -> tuple[tuple[tuple[str, dict[str, object]], ...], litellm.BadRequestError | None]:
|
||||
respond: Final = _StrictToolsLimitRejectionFirst(stream)
|
||||
kwargs: Final = {
|
||||
"model": "bedrock/us.anthropic.claude-sonnet-4-6",
|
||||
"messages": [{"role": "user", "content": "Find invoice 42"}],
|
||||
"tools": [tool],
|
||||
"aws_access_key_id": "AKIASTRICTTOOLS",
|
||||
"aws_secret_access_key": "strict-tools-secret",
|
||||
"aws_region_name": "us-east-1",
|
||||
"num_retries": 0,
|
||||
"drop_params": drop_params,
|
||||
"stream": stream != "off",
|
||||
"fake_stream": stream == "fake",
|
||||
}
|
||||
try:
|
||||
chunks: Final = (
|
||||
await _acompletion_chunks(AsyncHTTPHandler(transport=httpx.MockTransport(respond)), kwargs, stream)
|
||||
if asynchronous
|
||||
else _completion_chunks(
|
||||
HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), kwargs, stream
|
||||
)
|
||||
)
|
||||
except litellm.BadRequestError as error:
|
||||
return respond.sent, error
|
||||
assert chunks
|
||||
return respond.sent, None
|
||||
|
||||
|
||||
async def _acompletion_chunks(
|
||||
client: AsyncHTTPHandler, kwargs: Mapping[str, object], stream: Literal["off", "fake", "real"]
|
||||
) -> tuple[object, ...]:
|
||||
response: Final = await litellm.acompletion(client=client, **kwargs)
|
||||
return tuple([chunk async for chunk in response]) if stream == "fake" else (response,)
|
||||
|
||||
|
||||
def _completion_chunks(
|
||||
client: HTTPHandler, kwargs: Mapping[str, object], stream: Literal["off", "fake", "real"]
|
||||
) -> tuple[object, ...]:
|
||||
response: Final = litellm.completion(client=client, **kwargs)
|
||||
return tuple(response) if stream == "fake" else (response,)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
@pytest.mark.parametrize("stream", ["off", "fake", "real"], ids=["completion", "fake_stream", "stream"])
|
||||
async def test_bedrock_strict_tools_limit_rejection_is_sent_again_without_strict_under_drop_params(
|
||||
asynchronous: bool, stream: Literal["off", "fake", "real"], local_model_cost_map: None
|
||||
) -> None:
|
||||
sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream, drop_params=True)
|
||||
|
||||
assert error is None
|
||||
(first_path, first_body), (second_path, second_body) = sent
|
||||
first_tool_spec: Final = first_body["toolConfig"]["tools"][0]["toolSpec"]
|
||||
assert first_tool_spec["strict"] is True
|
||||
assert first_path == second_path == ("converse-stream" if stream == "real" else "converse")
|
||||
assert second_body == {
|
||||
**first_body,
|
||||
"toolConfig": {
|
||||
**first_body["toolConfig"],
|
||||
"tools": [{"toolSpec": {key: value for key, value in first_tool_spec.items() if key != "strict"}}],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
async def test_bedrock_strict_tools_limit_rejection_reaches_the_caller_without_drop_params(
|
||||
asynchronous: bool, local_model_cost_map: None
|
||||
) -> None:
|
||||
sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream="off", drop_params=None)
|
||||
|
||||
assert error is not None and "Too many strict tools" in str(error)
|
||||
assert len(sent) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
async def test_bedrock_strict_tools_limit_rejection_is_not_sent_again_when_no_tool_sent_strict(
|
||||
asynchronous: bool, local_model_cost_map: None
|
||||
) -> None:
|
||||
sent, error = await _converse_with_a_strict_tools_limit_rejection_first(
|
||||
asynchronous,
|
||||
stream="off",
|
||||
drop_params=True,
|
||||
tool={"type": "function", "function": {**_LOOKUP_FUNCTION, "strict": False}},
|
||||
)
|
||||
|
||||
assert error is not None and "Too many strict tools" in str(error)
|
||||
assert len(sent) == 1
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue