mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(bedrock): send an effort without thinking only through the bedrock provider
A Bedrock ARN served by another provider such as litellm_proxy matched the Bedrock Claude check by name alone, so its next hop got output_config it never got before. The shadow eval logger's own strip is no longer needed, since its translation names no provider
This commit is contained in:
parent
ce02ebfbd0
commit
14145a88c5
3 changed files with 41 additions and 4 deletions
|
|
@ -134,9 +134,7 @@ def _chat_request_from_anthropic_messages(
|
|||
)
|
||||
)
|
||||
translated, _ = adapter.translate_anthropic_to_openai(request) # pyright: ignore[reportArgumentType] # wire-body mapping is the surface's native request shape; the adapter is duck-typed and read-only here
|
||||
if request.get("thinking"):
|
||||
return translated
|
||||
return MappingProxyType({k: v for k, v in translated.items() if k != "output_config"})
|
||||
return translated
|
||||
|
||||
|
||||
def _chat_request_from_responses(
|
||||
|
|
|
|||
|
|
@ -1215,6 +1215,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
self,
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
new_kwargs: ChatCompletionRequest,
|
||||
custom_llm_provider: str | None,
|
||||
drop_params: bool,
|
||||
base_model: str | None,
|
||||
) -> None:
|
||||
|
|
@ -1224,7 +1225,8 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
it travels on its own instead of only alongside ``thinking``. ``format`` stays behind, since
|
||||
``_translate_output_format_to_openai`` carries it as ``response_format``. Without ``thinking``
|
||||
only ``effort`` goes, since the other keys (``task_budget``) never reached Converse there and
|
||||
it rejects them.
|
||||
it rejects them, and only through the ``bedrock`` provider, since an ARN-named model on another
|
||||
(``litellm_proxy``) never got it there.
|
||||
"""
|
||||
if "output_config" not in anthropic_message_request or not self._is_bedrock_claude_target(new_kwargs["model"]):
|
||||
return
|
||||
|
|
@ -1234,6 +1236,8 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
return
|
||||
|
||||
has_thinking: Final = bool(anthropic_message_request.get("thinking"))
|
||||
if not has_thinking and custom_llm_provider != "bedrock":
|
||||
return
|
||||
effort_config: Final = {
|
||||
k: v for k, v in output_config.items() if k == "effort" or (has_thinking and k != "format")
|
||||
}
|
||||
|
|
@ -1353,6 +1357,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
self._translate_output_config_to_openai(
|
||||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
drop_params=drop_params,
|
||||
base_model=base_model,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5625,6 +5625,40 @@ async def test_bedrock_application_inference_profile_loses_the_effort_only_when_
|
|||
assert sent.get("output_config") == ({"effort": "high"} if forwards_effort else None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_bedrock_arn_behind_another_provider_gets_no_effort_without_a_thinking_block(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
next_hop: Final = respx_mock.post("http://next-hop.test/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-next-hop",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": _APPLICATION_INFERENCE_PROFILE,
|
||||
"choices": [
|
||||
{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "Paris"}}
|
||||
],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
await litellm.anthropic.messages.acreate(
|
||||
model=f"litellm_proxy/{_APPLICATION_INFERENCE_PROFILE}",
|
||||
api_base="http://next-hop.test",
|
||||
api_key="sk-next-hop",
|
||||
max_tokens=64,
|
||||
messages=[{"role": "user", "content": "Name the capital of France."}],
|
||||
output_config={"effort": "low"},
|
||||
)
|
||||
|
||||
assert "output_config" not in json.loads(next_hop.calls.last.request.read())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"output_config",
|
||||
[pytest.param(None, id="null"), pytest.param({"format": _CITY_FORMAT}, id="format_only")],
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue