mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
fix(bedrock): map Anthropic batch row params the way real time does (#43087)
* fix(bedrock): map Anthropic batch row params the way real time does * fix(bedrock): let a batch row's allowed_openai_params reach the mapper * test(bedrock): assert the batch thinking value matches the real-time mapping * fix(bedrock): keep json_mode out of Anthropic batch rows and pin route-prefixed deployments --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
1987133b4e
commit
a76f23ac4f
2 changed files with 196 additions and 8 deletions
|
|
@ -58,8 +58,8 @@ from litellm.types.llms.openai import (
|
|||
OpenAIFileObject,
|
||||
PathLike,
|
||||
)
|
||||
from litellm.types.utils import ExtractedFileData, LlmProviders, SpecialEnums
|
||||
from litellm.utils import get_llm_provider
|
||||
from litellm.types.utils import ExtractedFileData, LlmProviders, SpecialEnums, all_litellm_params
|
||||
from litellm.utils import get_llm_provider, get_optional_params
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM
|
||||
from ..common_utils import (
|
||||
|
|
@ -88,6 +88,14 @@ def _frozen_mapping(items: Iterable[tuple[str, object]]) -> Mapping[str, object]
|
|||
return MappingProxyType(dict(items))
|
||||
|
||||
|
||||
_LITELLM_PARAMS_THE_MAPPER_TAKES: Final = frozenset({"allowed_openai_params"})
|
||||
_MAPPED_PARAMS_THE_REQUEST_HANDLER_STRIPS: Final = frozenset({"json_mode"})
|
||||
|
||||
|
||||
def _invoke_route_model(model: str) -> str:
|
||||
return f"invoke/{_strip_llm_routing_prefix(model).removeprefix('invoke/')}"
|
||||
|
||||
|
||||
def _strip_llm_routing_prefix(model: str) -> str:
|
||||
try:
|
||||
stripped_model, _, _, _ = get_llm_provider(model=model, custom_llm_provider=None)
|
||||
|
|
@ -891,16 +899,24 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
)
|
||||
|
||||
config: Final = AmazonAnthropicClaudeConfig()
|
||||
mapped_params = config.map_openai_params(
|
||||
non_default_params={},
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
drop_params=False,
|
||||
mapped_params = get_optional_params(
|
||||
model=_invoke_route_model(model),
|
||||
custom_llm_provider="bedrock",
|
||||
messages=messages,
|
||||
**MappingProxyType(
|
||||
{
|
||||
k: v
|
||||
for k, v in optional_params.items()
|
||||
if k not in all_litellm_params or k in _LITELLM_PARAMS_THE_MAPPER_TAKES
|
||||
}
|
||||
),
|
||||
)
|
||||
return config.transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params=mapped_params,
|
||||
optional_params={
|
||||
k: v for k, v in mapped_params.items() if k not in _MAPPED_PARAMS_THE_REQUEST_HANDLER_STRIPS
|
||||
},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from botocore.auth import S3SigV4Auth, SigV4Auth
|
|||
from botocore.awsrequest import AWSRequest
|
||||
from botocore.credentials import Credentials
|
||||
|
||||
from litellm.constants import DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET
|
||||
from litellm.llms.bedrock.files.transformation import BedrockJsonlFilesTransformation
|
||||
|
||||
|
||||
|
|
@ -1901,6 +1902,177 @@ class TestBedrockBatchNonChatEndpointRecords:
|
|||
]
|
||||
|
||||
|
||||
class TestBedrockBatchAnthropicRowParams:
|
||||
"""Anthropic batch rows get the OpenAI-to-Anthropic param mapping a real-time request gets.
|
||||
|
||||
Bedrock batch `modelInput` is the InvokeModel body, so a row's OpenAI params
|
||||
(`tools`, `reasoning_effort`, `max_tokens`, ...) have to be mapped the way
|
||||
`get_optional_params` maps them for `bedrock/invoke/...` at request time.
|
||||
Before that, the Anthropic branch wrote the row params into the body as
|
||||
sent, and Bedrock failed every record carrying a function tool
|
||||
(`tool type 'function' is not supported`) or a reasoning tier
|
||||
(`reasoning_effort: Extra inputs are not permitted`).
|
||||
"""
|
||||
|
||||
MODEL = "bedrock/us.anthropic.claude-sonnet-4-6"
|
||||
PARAMETERS = {"type": "object", "properties": {"city": {"type": "string"}}}
|
||||
|
||||
def _transform(self, url: str, body: dict, model: str = MODEL, target_model: str = "") -> dict:
|
||||
from litellm.llms.bedrock.files.transformation import BedrockFilesConfig
|
||||
|
||||
record = {"custom_id": "row-1", "method": "POST", "url": url, "body": {"model": model, **body}}
|
||||
result = BedrockFilesConfig()._transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||
[record], target_model=target_model
|
||||
)
|
||||
assert len(result) == 1
|
||||
return result[0]["modelInput"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url", "body"),
|
||||
[
|
||||
(
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"messages": [{"role": "user", "content": "Weather in Paris?"}],
|
||||
"tools": [{"type": "function", "function": {"name": "get_weather", "parameters": PARAMETERS}}],
|
||||
},
|
||||
),
|
||||
(
|
||||
"/v1/responses",
|
||||
{
|
||||
"input": "Weather in Paris?",
|
||||
"tools": [{"type": "function", "name": "get_weather", "parameters": PARAMETERS}],
|
||||
},
|
||||
),
|
||||
],
|
||||
ids=["chat", "responses"],
|
||||
)
|
||||
def test_function_tools_become_anthropic_tools(self, url, body):
|
||||
model_input = self._transform(url, body)
|
||||
|
||||
(tool,) = model_input["tools"]
|
||||
assert (tool["name"], tool["input_schema"]) == ("get_weather", self.PARAMETERS)
|
||||
assert "function" not in tool
|
||||
assert tool.get("type") != "function"
|
||||
|
||||
@pytest.mark.parametrize("route_prefix", ["converse/", "invoke/"])
|
||||
@pytest.mark.parametrize(
|
||||
"deployment_model",
|
||||
["bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "bedrock/us.anthropic.claude-sonnet-4-6"],
|
||||
ids=["budget", "adaptive"],
|
||||
)
|
||||
def test_route_prefixed_deployment_maps_like_the_plain_one(self, route_prefix, deployment_model):
|
||||
body = {
|
||||
"messages": [{"role": "user", "content": "Weather in Paris?"}],
|
||||
"tools": [{"type": "function", "function": {"name": "get_weather", "parameters": self.PARAMETERS}}],
|
||||
"reasoning_effort": "low",
|
||||
"max_tokens": 2048,
|
||||
}
|
||||
prefixed_model = deployment_model.replace("bedrock/", f"bedrock/{route_prefix}", 1)
|
||||
|
||||
plain = self._transform("/v1/chat/completions", body, model="claude-batch", target_model=deployment_model)
|
||||
prefixed = self._transform("/v1/chat/completions", body, model="claude-batch", target_model=prefixed_model)
|
||||
|
||||
assert prefixed == plain
|
||||
assert "input_schema" in prefixed["tools"][0]
|
||||
assert "reasoning_effort" not in prefixed
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url", "body"),
|
||||
[
|
||||
("/v1/chat/completions", {"messages": [{"role": "user", "content": "17 * 23?"}], "reasoning_effort": "low"}),
|
||||
("/v1/responses", {"input": "17 * 23?", "reasoning": {"effort": "low"}}),
|
||||
],
|
||||
ids=["chat", "responses"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("model", "expected_tier"),
|
||||
[
|
||||
(
|
||||
"bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
{"thinking": {"type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET}},
|
||||
),
|
||||
("bedrock/us.anthropic.claude-sonnet-4-6", {"output_config": {"effort": "low"}}),
|
||||
],
|
||||
ids=["budget", "adaptive"],
|
||||
)
|
||||
def test_reasoning_effort_becomes_thinking(self, model, expected_tier, url, body):
|
||||
from litellm.utils import get_optional_params
|
||||
|
||||
model_input = self._transform(url, body, model=model)
|
||||
real_time = get_optional_params(
|
||||
model=model.removeprefix("bedrock/"),
|
||||
custom_llm_provider="bedrock",
|
||||
messages=[{"role": "user", "content": "17 * 23?"}],
|
||||
reasoning_effort="low",
|
||||
)
|
||||
|
||||
assert "reasoning_effort" not in model_input
|
||||
assert {k: model_input[k] for k in expected_tier} == expected_tier
|
||||
assert (model_input["thinking"], model_input.get("output_config")) == (
|
||||
real_time["thinking"],
|
||||
real_time.get("output_config"),
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response_format",
|
||||
[
|
||||
{"type": "json_object"},
|
||||
{"type": "json_schema", "json_schema": {"name": "weather", "schema": PARAMETERS}},
|
||||
],
|
||||
ids=["json_object", "json_schema"],
|
||||
)
|
||||
def test_response_format_rows_keep_json_mode_out_of_the_body(self, response_format):
|
||||
model_input = self._transform(
|
||||
"/v1/chat/completions",
|
||||
{"messages": [{"role": "user", "content": "Weather in Paris?"}], "response_format": response_format},
|
||||
)
|
||||
|
||||
assert "json_mode" not in model_input
|
||||
assert "response_format" not in model_input
|
||||
assert ("output_config" in model_input) == (response_format["type"] == "json_schema")
|
||||
|
||||
def test_provider_native_params_still_pass_through(self):
|
||||
model_input = self._transform(
|
||||
"/v1/chat/completions",
|
||||
{"messages": [{"role": "user", "content": "hi"}], "max_tokens": 20, "top_k": 5},
|
||||
)
|
||||
|
||||
assert (model_input["top_k"], model_input["max_tokens"]) == (5, 20)
|
||||
|
||||
def test_unsupported_openai_param_fails_the_row_like_real_time(self):
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
|
||||
with pytest.raises(UnsupportedParamsError, match="logprobs"):
|
||||
self._transform("/v1/chat/completions", {"messages": [{"role": "user", "content": "hi"}], "logprobs": True})
|
||||
|
||||
def test_row_level_drop_params_drops_the_unsupported_param(self):
|
||||
model_input = self._transform(
|
||||
"/v1/chat/completions",
|
||||
{"messages": [{"role": "user", "content": "hi"}], "logprobs": True, "drop_params": True},
|
||||
)
|
||||
|
||||
assert "logprobs" not in model_input
|
||||
assert "drop_params" not in model_input
|
||||
|
||||
def test_row_level_allowed_openai_params_keeps_the_param(self):
|
||||
model_input = self._transform(
|
||||
"/v1/chat/completions",
|
||||
{"messages": [{"role": "user", "content": "hi"}], "logprobs": True, "allowed_openai_params": ["logprobs"]},
|
||||
)
|
||||
|
||||
assert model_input["logprobs"] is True
|
||||
assert "allowed_openai_params" not in model_input
|
||||
|
||||
def test_chat_record_metadata_stays_out_of_the_body(self):
|
||||
model_input = self._transform(
|
||||
"/v1/chat/completions",
|
||||
{"messages": [{"role": "user", "content": "hi"}], "metadata": {"tenant": "acct-1"}},
|
||||
)
|
||||
|
||||
assert "metadata" not in model_input
|
||||
|
||||
|
||||
class TestBedrockFileDeletion:
|
||||
S3_URI: Final = "s3://my-bucket/litellm-bedrock-files-model-abc.jsonl"
|
||||
URL: Final = "https://s3.us-west-2.amazonaws.com/my-bucket/litellm-bedrock-files-model-abc.jsonl"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue