mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-10 08:28:25 +00:00
fix(anthropic): strip beta-gated fields on Foundry chat and Bedrock chat Invoke under drop_params
Foundry chat and Bedrock chat Invoke now drop beta-gated body fields the same way the other legs do when drop_params is on and the request does not send the gating beta. The mid-conversation system normalizer validates only on the conversion path, and the strip WARNING names the beta the caller can send to keep the field.
This commit is contained in:
parent
8983631cc5
commit
e2552fe052
8 changed files with 153 additions and 32 deletions
|
|
@ -292,10 +292,20 @@ def log_beta_gated_strips(stripped: Sequence[str], model: str | None, provider:
|
|||
if not stripped:
|
||||
return
|
||||
verbose_logger.warning(
|
||||
"Stripped %s from the request for model=%s provider=%s because this leg does not carry the "
|
||||
"anthropic-beta that gates them (see anthropic_beta_headers_config.json). The request is sent "
|
||||
"without those fields instead of failing with a provider 400.",
|
||||
"Stripped %s from the request for model=%s provider=%s because drop_params is on and the request "
|
||||
"does not send the anthropic-beta that gates them on this provider (see anthropic_beta_headers_config.json). "
|
||||
"Send that beta to keep them where the provider accepts it.",
|
||||
sorted(set(stripped)),
|
||||
model,
|
||||
provider,
|
||||
)
|
||||
|
||||
|
||||
def strip_beta_gated_request_body(
|
||||
request: Mapping[str, object], headers: object, *, drop_params: object, model: str | None, provider: str
|
||||
) -> Mapping[str, object]:
|
||||
gated: Final = strip_beta_gated_request_fields(
|
||||
request, active_beta_values(headers, request.get("anthropic_beta") or ()), drop_params=drop_params
|
||||
)
|
||||
log_beta_gated_strips(gated.stripped, model=model, provider=provider)
|
||||
return gated.request
|
||||
|
|
|
|||
|
|
@ -17,11 +17,7 @@ import litellm
|
|||
import litellm.litellm_core_utils
|
||||
import litellm.types
|
||||
import litellm.types.utils
|
||||
from litellm.anthropic_beta_gated_fields import (
|
||||
active_beta_values,
|
||||
log_beta_gated_strips,
|
||||
strip_beta_gated_request_fields,
|
||||
)
|
||||
from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body
|
||||
from litellm.anthropic_beta_headers_manager import (
|
||||
update_request_with_filtered_beta,
|
||||
)
|
||||
|
|
@ -515,13 +511,15 @@ class AnthropicChatCompletion(BaseLLM):
|
|||
request_data=request_data,
|
||||
provider=custom_llm_provider,
|
||||
)
|
||||
gated: Final = strip_beta_gated_request_fields(
|
||||
filtered_data,
|
||||
active_beta_values(request_headers, filtered_data.get("anthropic_beta") or ()),
|
||||
drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the handler's bare dict
|
||||
data: Final = dict(
|
||||
strip_beta_gated_request_body(
|
||||
filtered_data,
|
||||
request_headers,
|
||||
drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the handler's bare dict
|
||||
model=model,
|
||||
provider=custom_llm_provider,
|
||||
)
|
||||
)
|
||||
log_beta_gated_strips(gated.stripped, model=model, provider=custom_llm_provider)
|
||||
data: Final = dict(gated.request)
|
||||
logging_obj.pre_call(
|
||||
input=messages,
|
||||
api_key=api_key,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ from pydantic import TypeAdapter
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.anthropic_beta_gated_fields import (
|
||||
StrippedMessages,
|
||||
log_beta_gated_strips,
|
||||
strip_beta_gated_native_request,
|
||||
strip_beta_gated_system_messages,
|
||||
|
|
@ -228,6 +227,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
else:
|
||||
return system_param
|
||||
|
||||
def _strip_beta_gated_system_turns(
|
||||
self, messages: Sequence[object], model: str, drop_params: object
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
validated: Final = _MESSAGE_MAPPINGS.validate_python(messages)
|
||||
gated: Final = strip_beta_gated_system_messages(validated, frozenset(), drop_params=drop_params)
|
||||
log_beta_gated_strips(gated.stripped, model=model, provider=self.custom_llm_provider or "anthropic")
|
||||
return _MESSAGE_MAPPINGS.validate_python(gated.messages) if gated.stripped else validated
|
||||
|
||||
def _normalize_system_role_messages(
|
||||
self, anthropic_messages_request: dict, model: str, drop_params: object
|
||||
) -> None:
|
||||
|
|
@ -271,22 +278,22 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
len(messages),
|
||||
)
|
||||
hoisted: Final = messages[:leading_count]
|
||||
keeps_system_turns: Final = supports_factory(
|
||||
model=model,
|
||||
custom_llm_provider=self.custom_llm_provider,
|
||||
key="supports_mid_conversation_system",
|
||||
)
|
||||
trailing_messages: Final = _MESSAGE_MAPPINGS.validate_python(messages[leading_count:]) # pyright: ignore[reportUnknownArgumentType] # messages comes from the bare request dict
|
||||
gated: Final = (
|
||||
StrippedMessages(messages=trailing_messages, stripped=())
|
||||
if keeps_system_turns
|
||||
else strip_beta_gated_system_messages(trailing_messages, frozenset(), drop_params=drop_params)
|
||||
)
|
||||
log_beta_gated_strips(gated.stripped, model=model, provider=self.custom_llm_provider or "anthropic")
|
||||
remaining: Final = (
|
||||
messages[leading_count:]
|
||||
if keeps_system_turns
|
||||
else list(convert_mid_conversation_system_turns(_MESSAGE_MAPPINGS.validate_python(gated.messages)))
|
||||
if supports_factory(
|
||||
model=model,
|
||||
custom_llm_provider=self.custom_llm_provider,
|
||||
key="supports_mid_conversation_system",
|
||||
)
|
||||
else list(
|
||||
convert_mid_conversation_system_turns(
|
||||
self._strip_beta_gated_system_turns(
|
||||
messages[leading_count:], # pyright: ignore[reportUnknownArgumentType] # messages comes from the bare request dict
|
||||
model=model,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
if hoisted or remaining != messages:
|
||||
anthropic_messages_request["messages"] = remaining
|
||||
|
|
|
|||
|
|
@ -4,11 +4,12 @@ Azure Anthropic handler - reuses AnthropicChatCompletion logic with Azure authen
|
|||
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion
|
||||
from litellm.llms.anthropic.chat.transformation import JsonModeRequest
|
||||
|
|
@ -75,13 +76,22 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion):
|
|||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
data: Final = config.transform_request(
|
||||
request_body: Final[Mapping[str, object]] = config.transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
data: Final = dict(
|
||||
strip_beta_gated_request_body(
|
||||
request_body,
|
||||
headers, # pyright: ignore[reportUnknownArgumentType] # validate_environment returns a bare dict
|
||||
drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the handler's bare dict
|
||||
model=model,
|
||||
provider=custom_llm_provider,
|
||||
)
|
||||
)
|
||||
|
||||
## LOGGING
|
||||
logging_obj.pre_call(
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final
|
|||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.anthropic_beta_gated_fields import strip_beta_gated_request_body
|
||||
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
|
|
@ -210,7 +211,15 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
if beta_list:
|
||||
_anthropic_request["anthropic_beta"] = beta_list
|
||||
|
||||
return _anthropic_request
|
||||
return dict(
|
||||
strip_beta_gated_request_body(
|
||||
_anthropic_request, # pyright: ignore[reportUnknownArgumentType] # built by the base signature's bare dict transform
|
||||
{},
|
||||
drop_params=litellm_params.get("drop_params"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is the base signature's bare dict
|
||||
model=model,
|
||||
provider="bedrock_invoke",
|
||||
)
|
||||
)
|
||||
|
||||
@property
|
||||
def uses_async_transform_request(self) -> bool:
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ sys.path.insert(
|
|||
)
|
||||
|
||||
import json
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -272,3 +273,40 @@ def test_completion_surfaces_the_status_and_headers_of_a_rejected_request():
|
|||
assert rejected.value.status_code == 400
|
||||
assert rejected.value.litellm_response_headers["x-request-id"] == "req-1"
|
||||
assert "max_tokens too large" in str(rejected.value)
|
||||
|
||||
|
||||
@respx.mock
|
||||
@pytest.mark.parametrize("drop_params", [True, False])
|
||||
def test_completion_drops_block_binding_without_its_beta_only_under_drop_params(
|
||||
drop_params: bool, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
route: Final = respx.post("https://example-resource.services.ai.azure.com/anthropic/v1/messages").mock(
|
||||
return_value=Response(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_bound",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"content": [{"type": "text", "text": "Hello"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1},
|
||||
},
|
||||
)
|
||||
)
|
||||
block_binding: Final = {"prefix_mismatch_behavior": "error"}
|
||||
|
||||
litellm.completion(
|
||||
model="azure_ai/claude-sonnet-4-6",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_base="https://example-resource.services.ai.azure.com/anthropic",
|
||||
api_key="azure-test-key",
|
||||
thinking={"type": "enabled", "budget_tokens": 1024, "block_binding": block_binding},
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert sent["thinking"]["budget_tokens"] == 1024
|
||||
assert ("block_binding" in sent["thinking"]) is not drop_params
|
||||
|
|
|
|||
|
|
@ -2008,3 +2008,32 @@ def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_co
|
|||
assert not AmazonAnthropicClaudeConfig().uses_opt_in_native_output(
|
||||
model, optional_params, {"model_info": model_info}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config")
|
||||
@pytest.mark.parametrize(
|
||||
("drop_params", "headers", "keeps_block_binding"),
|
||||
(
|
||||
(True, {}, False),
|
||||
(False, {}, True),
|
||||
(True, {"anthropic-beta": "thinking-binding-controls-2026-08-01"}, True),
|
||||
),
|
||||
ids=("drop-params-no-beta", "no-drop-params", "drop-params-with-beta"),
|
||||
)
|
||||
def test_bedrock_chat_invoke_drops_block_binding_only_when_drop_params_and_no_beta_gates_it(
|
||||
drop_params: bool, headers: dict[str, str], keeps_block_binding: bool, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
result: Final = AmazonAnthropicClaudeConfig().transform_request(
|
||||
model="us.anthropic.claude-sonnet-4-6",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
optional_params={
|
||||
"max_tokens": 2048,
|
||||
"thinking": {"type": "enabled", "budget_tokens": 1024, "block_binding": {"prefix_mismatch_behavior": "error"}},
|
||||
},
|
||||
litellm_params={"drop_params": drop_params},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
assert result["thinking"]["budget_tokens"] == 1024
|
||||
assert ("block_binding" in result["thinking"]) is keeps_block_binding
|
||||
|
|
|
|||
|
|
@ -4270,3 +4270,23 @@ def test_bedrock_messages_drops_tool_changes_from_a_system_turn_an_older_model_g
|
|||
assert any(block["type"] == "tool_addition" for block in blocks) is not drop_params
|
||||
assert {"type": "text", "text": "A weather tool is now available"} in blocks
|
||||
assert all(message.role != "system" for message in sent)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config")
|
||||
@pytest.mark.parametrize("drop_params", (True, False))
|
||||
def test_bedrock_messages_sends_the_binding_beta_with_block_binding_whatever_drop_params(drop_params: bool) -> None:
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request(
|
||||
model="us.anthropic.claude-sonnet-4-6",
|
||||
messages=[{"role": "user", "content": "Reply with OK"}],
|
||||
anthropic_messages_optional_request_params={
|
||||
"max_tokens": 2048,
|
||||
"thinking": {"type": "enabled", "budget_tokens": 1024, "block_binding": BLOCK_BINDING},
|
||||
},
|
||||
litellm_params=GenericLiteLLMParams(drop_params=drop_params),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert result["thinking"]["block_binding"] == BLOCK_BINDING
|
||||
assert "thinking-binding-controls-2026-08-01" in result["anthropic_beta"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue