diff --git a/litellm/anthropic_beta_gated_fields.py b/litellm/anthropic_beta_gated_fields.py index 983700f113b..730395f2d66 100644 --- a/litellm/anthropic_beta_gated_fields.py +++ b/litellm/anthropic_beta_gated_fields.py @@ -196,11 +196,17 @@ def _strip_beta_gated_message(message: object, active_betas: Set[str]) -> Stripp return _strip_beta_gated_system_message(mapping, active_betas) +def _ends_on_assistant(messages: Sequence[object]) -> bool: + last: Final = _as_mapping(messages[-1]) if messages else None + return last is not None and last.get("role") == "assistant" + + def strip_beta_gated_system_messages( messages: Sequence[object], active_betas: Set[str], *, drop_params: object ) -> StrippedMessages: """Drop the gated fields of every mid-conversation ``role: system`` message. Returns the same - sequence when nothing is gated.""" + sequence when nothing is gated, or when dropping an emptied system turn would leave the conversation + ending on an assistant turn, which the model would read as a prefill to continue.""" if not drops_unsupported_params(drop_params): return StrippedMessages(messages=messages, stripped=()) per_message: Final = tuple(_strip_beta_gated_message(message, active_betas) for message in messages) @@ -209,12 +215,12 @@ def strip_beta_gated_system_messages( ) # comprehension-ok: flattening per-message strip reports if not stripped: return StrippedMessages(messages=messages, stripped=()) - return StrippedMessages( - messages=[ - message for result in per_message for message in result.messages - ], # comprehension-ok: flattening per-message results - stripped=stripped, - ) + kept: Final = [ + message for result in per_message for message in result.messages + ] # comprehension-ok: flattening per-message results + if _ends_on_assistant(kept) and not _ends_on_assistant(messages): + return StrippedMessages(messages=messages, stripped=()) + return StrippedMessages(messages=kept, stripped=stripped) def strip_beta_gated_request_fields( @@ -291,12 +297,18 @@ def strip_beta_gated_native_request( def log_beta_gated_strips(stripped: Sequence[str], model: str | None, provider: str) -> None: if not stripped: return + described: Final = ", ".join( + f"{field} (unlocked by anthropic-beta {' or '.join(sorted(_FIELD_BETAS[field]))})" + if field in _FIELD_BETAS + else field + for field in sorted(set(stripped)) + ) verbose_logger.warning( "Dropped %s for model=%s provider=%s because drop_params is on and the provider would reject them on " - "this request, either since this provider leg does not send the anthropic-beta that unlocks them (see " + "this request, either since this provider leg does not send that anthropic-beta (see " "anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user " "messages.", - sorted(set(stripped)), + described, model, provider, ) diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index 24bdbcd5bbf..e5ced61d0c4 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -241,6 +241,10 @@ def get_provider_name(provider: str) -> str: return aliases.get(provider, provider) +def has_beta_headers_config(provider: str) -> bool: + return get_provider_name(provider) in _load_beta_headers_config() + + def filter_and_transform_beta_headers( beta_headers: list[str], provider: str, diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index 46f96b4267d..29b2beb1d7c 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -12,7 +12,11 @@ from litellm.anthropic_beta_gated_fields import ( strip_beta_gated_system_messages, unstripped_native_request, ) -from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config +from litellm.anthropic_beta_headers_manager import ( + get_unsupported_headers, + has_beta_headers_config, + is_beta_header_supported_by_bundled_config, +) from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -230,8 +234,10 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): self, messages: Sequence[object], model: str, drop_params: object ) -> tuple[Mapping[str, object], ...]: validated: Final = _MESSAGE_MAPPINGS.validate_python(messages) + if not has_beta_headers_config(self._resolved_provider): + return validated gated: Final = strip_beta_gated_system_messages(validated, frozenset(), drop_params=drop_params) - log_beta_gated_strips(gated.stripped, model=model, provider=self.custom_llm_provider or "anthropic") + log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider) return _MESSAGE_MAPPINGS.validate_python(gated.messages) if gated.stripped else validated def _normalize_system_role_messages( @@ -351,7 +357,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): model=model, ), ) - return self._finalize_messages_headers(headers, optional_params, messages), api_base + return self.finalize_messages_headers(headers, optional_params, messages), api_base async def avalidate_anthropic_messages_environment( self, @@ -391,7 +397,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): model=model, ), ) - return self._finalize_messages_headers(oauth_headers, optional_params, messages), api_base + return self.finalize_messages_headers(oauth_headers, optional_params, messages), api_base def _require_auth_header(self, auth_header: Mapping[str, str] | None, model: str) -> Mapping[str, str]: if auth_header is None: @@ -425,7 +431,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): return config_allows_workload_identity(self) - def _finalize_messages_headers( + def finalize_messages_headers( self, headers: dict, # mutable-ok: out-param optional_params: dict, # mutable-ok: out-param @@ -636,8 +642,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): def strips_beta_gated_fields_from_final_headers(self) -> bool: """False on legs whose final ``anthropic_beta`` list is only known after this transform (Bedrock Invoke), - which strip beta-gated fields themselves once that list exists.""" - return True + which strip beta-gated fields themselves once that list exists, and on providers with no row in + ``anthropic_beta_headers_config.json``, whose filter drops every beta and so cannot say which fields fail.""" + return has_beta_headers_config(self._resolved_provider) def transform_anthropic_messages_request( self, diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index 7a4c707860d..d3a4279f3bc 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -661,12 +661,13 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()), dict(options), ) - body: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( + config: Final = AnthropicMessagesConfig() + body: Final = config.transform_anthropic_messages_request( model=resolved_model, messages=strip_provider_specific_fields_from_anthropic_messages(messages), anthropic_messages_optional_request_params=filtered, litellm_params=GenericLiteLLMParams(), - headers={}, + headers=config.finalize_messages_headers({}, dict(filtered), list(messages)), # pyright: ignore[reportUnknownMemberType] # the shared helper takes and returns bare dicts ) return MappingProxyType(_JSON_OBJECT.validate_python(body)) except Exception: # noqa: BLE001 # an unsupported hypothetical request is unavailable, never an inference failure diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 31b031e1839..da723f20ee7 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1830,14 +1830,12 @@ class AmazonConverseConfig(BaseConfig): base_model: Final = BedrockModelInfo.get_base_model(model) header_beta_list: Final = get_anthropic_beta_from_headers(headers or {}) - body_beta_list: Final = sorted( + client_beta_list: Final = header_beta_list or sorted( active_beta_values( {}, additional_request_params.get("anthropic_beta"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # additional_request_params is a bare dict ) - - set(header_beta_list) ) - client_beta_list: Final = [*header_beta_list, *body_beta_list] eager_beta: Final = ( (ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,) if base_model.startswith("anthropic") diff --git a/litellm/llms/bedrock/claude_platform/transformation.py b/litellm/llms/bedrock/claude_platform/transformation.py index 0194f1c7e45..a5ecc37d394 100644 --- a/litellm/llms/bedrock/claude_platform/transformation.py +++ b/litellm/llms/bedrock/claude_platform/transformation.py @@ -72,6 +72,7 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): ), code_execution_tool_used=self.is_code_execution_tool_used(tools=optional_params.get("tools")), container_with_skills_used=self.is_container_with_skills_used(optional_params=optional_params), + thinking=optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict ) anthropic_headers["anthropic-workspace-id"] = workspace_id return {**headers, **anthropic_headers} diff --git a/litellm/llms/databricks/messages/transformation.py b/litellm/llms/databricks/messages/transformation.py index f5343445957..90b63d860d3 100644 --- a/litellm/llms/databricks/messages/transformation.py +++ b/litellm/llms/databricks/messages/transformation.py @@ -144,7 +144,7 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig) ("content-type" if key.lower() == "content-type" else key): value for key, value in {**auth_headers, **client_anthropic_headers}.items() } - finalized_headers: Final[dict[str, str]] = self._finalize_messages_headers( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared helper returns a bare dict # mutable-ok: becomes the outbound headers + finalized_headers: Final[dict[str, str]] = self.finalize_messages_headers( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared helper returns a bare dict # mutable-ok: becomes the outbound headers headers=normalized_headers, optional_params=optional_params, messages=messages, diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index 525c247590d..b28d37a0563 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -174,6 +174,12 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert if messages_carry_output_config(messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value) + beta_values.update( + anthropic_model_info.thinking_betas( + optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict + ) + ) + if beta_values: request_headers["anthropic-beta"] = ",".join(beta_values) diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index d88ca10fd04..90d0df2b113 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -501,3 +501,23 @@ def test_messages_request_drops_clear_at_from_a_mid_conversation_system_turn_wit ) assert request["messages"][1] == {"role": "system", "content": "be terse"} assert system_turn == {"role": "system", "content": "be terse", "clear_at": "end_turn"} + + +def test_a_provider_without_a_beta_headers_row_forwards_beta_gated_fields_under_drop_params( + local_beta_headers_config: None, +) -> None: + thinking: Final = {"type": "enabled", "budget_tokens": 1024, "display": "updates"} + request: Final = MinimaxMessagesConfig().transform_anthropic_messages_request( + model="MiniMax-M2", + messages=[ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "Hello"}, + {"role": "user", "content": "how are you"}, + {"role": "system", "content": "Answer in French", "output_config": {"effort": "low"}}, + ], + anthropic_messages_optional_request_params={"max_tokens": 64, "thinking": dict(thinking)}, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers={}, + ) + assert request["thinking"] == thinking + assert request["messages"][-1]["output_config"] == {"effort": "low"} diff --git a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py index 7a6b6534131..4d0ebef4f2d 100644 --- a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py +++ b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py @@ -13,7 +13,7 @@ import litellm from litellm.caching.dual_cache import DualCache from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.anthropic.count_tokens import handler as count_handler -from litellm.llms.anthropic.messages.native.transformation import DEFAULT_ANTHROPIC_API_VERSION +from litellm.llms.anthropic.messages.native.transformation import DEFAULT_ANTHROPIC_API_VERSION, AnthropicMessagesConfig from litellm.llms.anthropic.prompt_cache_prediction import ( CountedPromptCachePlan, NativePredictionTarget, @@ -26,6 +26,7 @@ from litellm.llms.anthropic.prompt_cache_prediction import ( parse_cache_plan, parse_observed_cache, parse_prompt, + prepare_native_baseline_body, resolve_baseline_prediction_target, resolve_prediction_target, supported_prediction_headers, @@ -36,7 +37,7 @@ from litellm.proxy import proxy_server from litellm.proxy.hooks.prompt_cache_prediction import PromptCacheObserver from litellm.proxy.management_endpoints.prompt_cache_prediction import predict_arm from litellm.proxy.utils import InternalUsageCache -from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo +from litellm.types.router import Deployment, GenericLiteLLMParams, LiteLLM_Params, ModelInfo from litellm.types.utils import CacheCreationTokenDetails, ModelResponse, PromptTokensDetailsWrapper, Usage _MODEL: Final = "claude-sonnet-5" @@ -50,9 +51,12 @@ def _body() -> dict[str, JsonValue]: "model": _MODEL, "system": "Keep this context", "tools": [{"name": "lookup", "input_schema": {"type": "object"}}], - "messages": [{"role": "user", "content": [ - {"type": "text", "text": "A cacheable prefix", "cache_control": {"type": "ephemeral"}} - ]}], + "messages": [ + { + "role": "user", + "content": [{"type": "text", "text": "A cacheable prefix", "cache_control": {"type": "ephemeral"}}], + } + ], } @@ -88,7 +92,8 @@ async def test_observer_records_only_version_supported_by_token_counter(version: "httpx_response": httpx.Response(200, request=wire), "first_api_call_start_time": datetime.fromtimestamp(1000.0), "standard_logging_object": { - "status": "success", "model_id": _DEPLOYMENT, + "status": "success", + "model_id": _DEPLOYMENT, "metadata": {"user_api_key_hash": _CALLER}, }, }, @@ -104,13 +109,16 @@ async def test_observer_records_only_version_supported_by_token_counter(version: assert await lookup(cache, other_scope, prefix, now=1010.0) is None -@pytest.mark.parametrize("headers, supported", [ - ({}, True), - ({"Anthropic-Version": DEFAULT_ANTHROPIC_API_VERSION}, True), - ({"anthropic-version": "2099-01-01"}, False), - ({"Anthropic-Beta": ""}, False), - ({"anthropic-beta": "future-feature"}, False), -]) +@pytest.mark.parametrize( + "headers, supported", + [ + ({}, True), + ({"Anthropic-Version": DEFAULT_ANTHROPIC_API_VERSION}, True), + ({"anthropic-version": "2099-01-01"}, False), + ({"Anthropic-Beta": ""}, False), + ({"anthropic-beta": "future-feature"}, False), + ], +) def test_prediction_header_eligibility(headers: Mapping[str, str], supported: bool) -> None: assert supported_prediction_headers(headers) is supported @@ -150,49 +158,89 @@ async def test_provider_count_uses_same_version_and_preserves_native_input( @pytest.mark.parametrize("source", ["static", "database"]) @pytest.mark.asyncio async def test_environment_credential_matches_native_count_and_observed_scope( - source: str, monkeypatch: pytest.MonkeyPatch, + source: str, + monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv("LIT7658_PROVIDER_KEY", _KEY) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) params: Final = { - "model": f"anthropic/{_MODEL}", "api_key": "os.environ/LIT7658_PROVIDER_KEY", + "model": f"anthropic/{_MODEL}", + "api_key": "os.environ/LIT7658_PROVIDER_KEY", "api_base": "https://api.anthropic.com", } - router: Final = litellm.Router(model_list=[{ - "model_name": "test-native", "litellm_params": dict(params), "model_info": {"id": _DEPLOYMENT}, - }] if source == "static" else [], num_retries=0) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "test-native", + "litellm_params": dict(params), + "model_info": {"id": _DEPLOYMENT}, + } + ] + if source == "static" + else [], + num_retries=0, + ) if source == "database": monkeypatch.setattr(proxy_server, "llm_router", router) - assert proxy_server.ProxyConfig()._add_deployment([SimpleNamespace( - model_id=_DEPLOYMENT, model_name="test-native", model_info={}, litellm_params=dict(params), - )]) == 1 + assert ( + proxy_server.ProxyConfig()._add_deployment( + [ + SimpleNamespace( + model_id=_DEPLOYMENT, + model_name="test-native", + model_info={}, + litellm_params=dict(params), + ) + ] + ) + == 1 + ) deployment: Final = router.get_deployment(_DEPLOYMENT) assert deployment is not None target: Final = resolve_prediction_target(deployment.litellm_params) assert isinstance(target, NativePredictionTarget) body: Final = _body() with respx.mock() as upstream: - native: Final = upstream.post("https://api.anthropic.com/v1/messages").respond(200, json={ - "id": "msg_test", "type": "message", "role": "assistant", "model": _MODEL, - "content": [{"type": "text", "text": "Hello"}], "stop_reason": "end_turn", "stop_sequence": None, - "usage": {"input_tokens": 11, "output_tokens": 1, "cache_read_input_tokens": 300}, - }) + native: Final = upstream.post("https://api.anthropic.com/v1/messages").respond( + 200, + json={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": _MODEL, + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 11, "output_tokens": 1, "cache_read_input_tokens": 300}, + }, + ) counter: Final = upstream.post("https://api.anthropic.com/v1/messages/count_tokens").respond( - 200, json={"input_tokens": 311}, + 200, + json={"input_tokens": 311}, ) await router.aanthropic_messages( - model="test-native", max_tokens=1, **{key: value for key, value in body.items() if key != "model"}, + model="test-native", + max_tokens=1, + **{key: value for key, value in body.items() if key != "model"}, ) assert await count_prompt_tokens(target.model, target.api_key, body) == 311 assert native.call_count == counter.call_count == 1 assert native.calls.last.request.headers["x-api-key"] == counter.calls.last.request.headers["x-api-key"] == _KEY - observed: Final = parse_observed_cache(native.calls.last.request, ModelResponse( - model=_MODEL, usage=Usage( - prompt_tokens=311, completion_tokens=1, total_tokens=312, - prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300), + observed: Final = parse_observed_cache( + native.calls.last.request, + ModelResponse( + model=_MODEL, + usage=Usage( + prompt_tokens=311, + completion_tokens=1, + total_tokens=312, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300), + ), ), - ), _CALLER, _DEPLOYMENT) + _CALLER, + _DEPLOYMENT, + ) assert observed is not None assert observed.scope == cache_scope(_CALLER, _DEPLOYMENT, target.api_key, target.model) @@ -200,15 +248,26 @@ async def test_environment_credential_matches_native_count_and_observed_scope( @pytest.mark.parametrize("inline_key", [None, _KEY]) @pytest.mark.asyncio async def test_named_credential_is_explicitly_unsupported_before_count( - inline_key: str | None, monkeypatch: pytest.MonkeyPatch, + inline_key: str | None, + monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(litellm, "credential_list", [CredentialItem( - credential_name="test-named", credential_info={}, credential_values={"api_key": "test-named-provider-key"}, - )]) + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem( + credential_name="test-named", + credential_info={}, + credential_values={"api_key": "test-named-provider-key"}, + ) + ], + ) deployment: Final = Deployment( model_name="test-native", litellm_params=LiteLLM_Params( - model=f"anthropic/{_MODEL}", api_key=inline_key, litellm_credential_name="test-named", + model=f"anthropic/{_MODEL}", + api_key=inline_key, + litellm_credential_name="test-named", ), model_info=ModelInfo(id=_DEPLOYMENT), ) @@ -232,8 +291,7 @@ def _cache_plan(body: Mapping[str, JsonValue]) -> PromptCachePlan: def _text(text: str, ttl: str | None = None) -> dict[str, JsonValue]: - return {"type": "text", "text": text, - **({"cache_control": {"type": "ephemeral", "ttl": ttl}} if ttl else {})} + return {"type": "text", "text": text, **({"cache_control": {"type": "ephemeral", "ttl": ttl}} if ttl else {})} def _prompt(*blocks: dict[str, JsonValue], role: str = "user", **options: JsonValue) -> dict[str, JsonValue]: @@ -251,16 +309,19 @@ def test_public_predictor_preserves_string_message_policy(text: str, supported: def test_cache_plan_preserves_hierarchical_prefixes_and_public_policy() -> None: body: Final = _prompt( - _text("First turn", "5m"), system=[_text("Stable instructions", "1h")], - tools=[{"name": "lookup", "input_schema": {"type": "object"}, - "cache_control": {"type": "ephemeral", "ttl": "1h"}}], + _text("First turn", "5m"), + system=[_text("Stable instructions", "1h")], + tools=[ + {"name": "lookup", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral", "ttl": "1h"}} + ], ) plan: Final = _cache_plan(body) changed: Final = _cache_plan({**body, "system": [_text("Changed instructions", "1h")]}) assert tuple(marker.ttl_seconds for marker in plan.breakpoints) == (3600, 3600, 300) assert plan.breakpoints[0].fingerprint == changed.breakpoints[0].fingerprint - assert all(left.fingerprint != right.fingerprint for left, right - in zip(plan.breakpoints[1:], changed.breakpoints[1:])) + assert all( + left.fingerprint != right.fingerprint for left, right in zip(plan.breakpoints[1:], changed.breakpoints[1:]) + ) assert plan.breakpoints[0].prefix_body == { "tools": [{"name": "lookup", "input_schema": {"type": "object"}}], "messages": [], @@ -268,51 +329,80 @@ def test_cache_plan_preserves_hierarchical_prefixes_and_public_policy() -> None: assert parse_prompt(body) is None -@pytest.mark.parametrize("kind, added, matches", [ - ("text", 19, True), ("text", 20, False), ("tool_use", 30, True), - ("tool_result", 30, True), -]) +@pytest.mark.parametrize( + "kind, added, matches", + [ + ("text", 19, True), + ("text", 20, False), + ("tool_use", 30, True), + ("tool_result", 30, True), + ], +) def test_cache_plan_lookback_counts_native_positions( - kind: str, added: int, matches: bool, + kind: str, + added: int, + matches: bool, ) -> None: previous: Final = _cache_plan(_body()) appended: Final[list[dict[str, JsonValue]]] = [ {"type": "tool_use", "id": f"tool_{index}", "name": "lookup", "input": {}} - if kind == "tool_use" else - {"type": "tool_result", "tool_use_id": f"tool_{index}", "content": "done"} - if kind == "tool_result" else - {"type": "text", "text": f"Added {index}"} + if kind == "tool_use" + else {"type": "tool_result", "tool_use_id": f"tool_{index}", "content": "done"} + if kind == "tool_result" + else {"type": "text", "text": f"Added {index}"} for index in range(added) ] - current: Final = _cache_plan({**_body(), **_prompt( - _text("A cacheable prefix"), *appended[:-1], - {**appended[-1], "cache_control": {"type": "ephemeral"}}, - )}) - assert (previous.breakpoints[0].fingerprint - in current.breakpoints[0].lookback_fingerprints) is matches + current: Final = _cache_plan( + { + **_body(), + **_prompt( + _text("A cacheable prefix"), + *appended[:-1], + {**appended[-1], "cache_control": {"type": "ephemeral"}}, + ), + } + ) + assert (previous.breakpoints[0].fingerprint in current.breakpoints[0].lookback_fingerprints) is matches -@pytest.mark.parametrize("change, same_prefix, same_content", [ - ("tool_order", False, False), ("effort", False, False), - ("standard_speed", True, True), ("ttl", False, True), -]) +@pytest.mark.parametrize( + "change, same_prefix, same_content", + [ + ("tool_order", False, False), + ("effort", False, False), + ("standard_speed", True, True), + ("ttl", False, True), + ], +) def test_cache_plan_identity_respects_settings_and_preserves_content( - change: str, same_prefix: bool, same_content: bool, + change: str, + same_prefix: bool, + same_content: bool, ) -> None: tool_input: Final[dict[str, JsonValue]] = {"a": 1, "b": 2, "cache_control": {"ttl": "user-data"}} block: Final[dict[str, JsonValue]] = { - "type": "tool_use", "id": "tool_1", "name": "lookup", "input": tool_input, + "type": "tool_use", + "id": "tool_1", + "name": "lookup", + "input": tool_input, "cache_control": {"type": "ephemeral", "ttl": "5m"}, } changed_block: Final = ( - {**block, "input": dict(reversed(tool_input.items()))} if change == "tool_order" else - {**block, "cache_control": {"type": "ephemeral", "ttl": "1h"}} if change == "ttl" else block + {**block, "input": dict(reversed(tool_input.items()))} + if change == "tool_order" + else {**block, "cache_control": {"type": "ephemeral", "ttl": "1h"}} + if change == "ttl" + else block ) before: Final = _cache_plan(_prompt(block, role="assistant", output_config={"effort": "low"})).breakpoints[0] - after: Final = _cache_plan(_prompt( - changed_block, role="assistant", output_config={"effort": "high" if change == "effort" else "low"}, - **({"speed": "standard"} if change == "standard_speed" else {}), - )).breakpoints[0] + after: Final = _cache_plan( + _prompt( + changed_block, + role="assistant", + output_config={"effort": "high" if change == "effort" else "low"}, + **({"speed": "standard"} if change == "standard_speed" else {}), + ) + ).breakpoints[0] assert (before.fingerprint == after.fingerprint) is same_prefix assert (before.fingerprint in after.lookback_fingerprints) is same_prefix assert (before.content_fingerprint == after.content_fingerprint) is same_content @@ -323,30 +413,41 @@ def test_cache_plan_identity_respects_settings_and_preserves_content( def test_cache_plan_automatic_cache_and_thinking_use_last_cacheable_block() -> None: body: Final = _prompt( - _text("A stable answer"), {"type": "thinking", "thinking": "Thinking", "signature": "signature"}, - role="assistant", thinking={"type": "adaptive"}, cache_control={"type": "ephemeral", "ttl": "1h"}, + _text("A stable answer"), + {"type": "thinking", "thinking": "Thinking", "signature": "signature"}, + role="assistant", + thinking={"type": "adaptive"}, + cache_control={"type": "ephemeral", "ttl": "1h"}, ) plan: Final = _cache_plan(body) assert len(plan.breakpoints) == 1 assert plan.breakpoints[0].ttl_seconds == 3600 assert plan.breakpoints[0].prefix_body == { "thinking": {"type": "adaptive"}, - "messages": [{"role": "assistant", "content": [ - {"type": "text", "text": "A stable answer"}, - ]}], + "messages": [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "A stable answer"}, + ], + } + ], } assert parse_prompt(body) is None -@pytest.mark.parametrize("body, reason", [ - (_prompt({"type": "image"}), "unsupported_prompt_shape"), - ({**_body(), "unknown_native_setting": True}, "unsupported_prompt_shape"), - ({**_body(), "cache_control": {"type": "ephemeral", "ttl": "1h"}}, - "conflicting_cache_ttl"), - (_prompt(_text("five", "5m"), _text("hour", "1h")), "invalid_cache_ttl_order"), -]) +@pytest.mark.parametrize( + "body, reason", + [ + (_prompt({"type": "image"}), "unsupported_prompt_shape"), + ({**_body(), "unknown_native_setting": True}, "unsupported_prompt_shape"), + ({**_body(), "cache_control": {"type": "ephemeral", "ttl": "1h"}}, "conflicting_cache_ttl"), + (_prompt(_text("five", "5m"), _text("hour", "1h")), "invalid_cache_ttl_order"), + ], +) def test_cache_plan_unsupported_is_explicit( - body: Mapping[str, JsonValue], reason: str, + body: Mapping[str, JsonValue], + reason: str, ) -> None: result: Final = parse_cache_plan(body) assert isinstance(result, UnsupportedCachePlan) @@ -354,30 +455,46 @@ def test_cache_plan_unsupported_is_explicit( @pytest.mark.asyncio -@pytest.mark.parametrize("model, counts, reason", [ - (None, (100, 150, 200), None), - (None, (100, 201, 200), "inconsistent_prefix_token_count"), - (None, (151, 150, 200), "inconsistent_prefix_token_count"), - (None, (None, 150, 200), "token_count_unavailable"), - ("claude-opus-5", (100, 150, 200), None), - ("claude-sonnet-5", (100, 150, 200), None), - ("declared-cache-model", (100, 150, 200), None), - ("unknown-cache-model", (100, 150, 200), "unsupported_thinking_cache_semantics"), - ("claude-haiku-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"), - ("claude-sonnet-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"), -]) +@pytest.mark.parametrize( + "model, counts, reason", + [ + (None, (100, 150, 200), None), + (None, (100, 201, 200), "inconsistent_prefix_token_count"), + (None, (151, 150, 200), "inconsistent_prefix_token_count"), + (None, (None, 150, 200), "token_count_unavailable"), + ("claude-opus-5", (100, 150, 200), None), + ("claude-sonnet-5", (100, 150, 200), None), + ("declared-cache-model", (100, 150, 200), None), + ("unknown-cache-model", (100, 150, 200), "unsupported_thinking_cache_semantics"), + ("claude-haiku-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"), + ("claude-sonnet-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"), + ], +) async def test_cache_plan_count_conserves_total_and_rejects_unknown( - model: str | None, counts: tuple[int | None, int | None, int], reason: str | None, + model: str | None, + counts: tuple[int | None, int | None, int], + reason: str | None, monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setitem(litellm.model_cost, "declared-cache-model", { - "litellm_provider": "anthropic", "mode": "chat", "supports_thinking_cache_preservation": True, - }) - plan: Final = _cache_plan(_prompt( - {"type": "thinking", "thinking": "Retained thought", "signature": "signature"} - if model else _text("first", "5m"), - _text("second", "5m"), _text("uncached"), role="assistant" if model else "user", - )) + monkeypatch.setitem( + litellm.model_cost, + "declared-cache-model", + { + "litellm_provider": "anthropic", + "mode": "chat", + "supports_thinking_cache_preservation": True, + }, + ) + plan: Final = _cache_plan( + _prompt( + {"type": "thinking", "thinking": "Retained thought", "signature": "signature"} + if model + else _text("first", "5m"), + _text("second", "5m"), + _text("uncached"), + role="assistant" if model else "user", + ) + ) async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None: assert reason != "unsupported_thinking_cache_semantics", "Unverified thinking retention must skip counting" @@ -399,12 +516,15 @@ async def test_cache_plan_count_conserves_total_and_rejects_unknown( @pytest.mark.parametrize("section", ["system", "tools"]) @pytest.mark.parametrize("rejects_prefix", (False, True)) async def test_native_count_preserves_settings_and_requires_every_prefix( - section: str, rejects_prefix: bool, monkeypatch: pytest.MonkeyPatch, + section: str, + rejects_prefix: bool, + monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) params: Final = LiteLLM_Params( - model=f"anthropic/{_MODEL}", api_key=_KEY, + model=f"anthropic/{_MODEL}", + api_key=_KEY, api_base="https://gateway.example/v1/messages", ) target: Final = resolve_baseline_prediction_target(params) @@ -413,19 +533,38 @@ async def test_native_count_preserves_settings_and_requires_every_prefix( assert not isinstance(resolve_prediction_target(params), NativePredictionTarget) body: Final = _body() marker: Final[dict[str, JsonValue]] = {"type": "ephemeral", "ttl": "1h"} - body[section] = ([_text("A cached system", "1h")] if section == "system" else [{ - "name": "lookup", "input_schema": {"type": "object"}, "cache_control": marker, - }]) - plan: Final = _cache_plan({**body, **_prompt( - _text("A later prefix", "5m"), _text("An uncached suffix"), - thinking={"type": "adaptive"}, tool_choice={"type": "auto"}, output_config={"effort": "high"}, - )}) + body[section] = ( + [_text("A cached system", "1h")] + if section == "system" + else [ + { + "name": "lookup", + "input_schema": {"type": "object"}, + "cache_control": marker, + } + ] + ) + plan: Final = _cache_plan( + { + **body, + **_prompt( + _text("A later prefix", "5m"), + _text("An uncached suffix"), + thinking={"type": "adaptive"}, + tool_choice={"type": "auto"}, + output_config={"effort": "high"}, + ), + } + ) assert len(plan.breakpoints) == 2 assert plan.breakpoints[0].prefix_body["messages"] == [] async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None: return await count_prompt_tokens( - model, api_key, {**body, "max_tokens": 100}, api_base=target.api_base, + model, + api_key, + {**body, "max_tokens": 100}, + api_base=target.api_base, ) with respx.mock(assert_all_called=False) as upstream: @@ -434,12 +573,16 @@ async def test_native_count_preserves_settings_and_requires_every_prefix( upstream.post(endpoint, json={**body, "model": _MODEL}).respond( 400 if rejects_prefix and index == 1 else 200, json={"detail": {"error": "messages parameter is required"}} - if rejects_prefix and index == 1 else {"input_tokens": tokens}, + if rejects_prefix and index == 1 + else {"input_tokens": tokens}, + ) + for index, (body, tokens) in enumerate( + ( + (plan.full_body, 6000), + (plan.breakpoints[0].prefix_body, 5000), + (plan.breakpoints[1].prefix_body, 5800), + ) ) - for index, (body, tokens) in enumerate(( - (plan.full_body, 6000), (plan.breakpoints[0].prefix_body, 5000), - (plan.breakpoints[1].prefix_body, 5800), - )) ) unexpected: Final = upstream.post(endpoint).respond(200, json={"input_tokens": 1}) result: Final = await count_cache_plan(target.model, target.api_key, plan, count) @@ -452,3 +595,32 @@ async def test_native_count_preserves_settings_and_requires_every_prefix( assert tuple(marker.prefix_tokens for marker in result.breakpoints) == (5000, 5800) assert tuple(route.call_count for route in routes) == (1, 1, 1) assert unexpected.call_count == 0 + + +def test_native_baseline_body_matches_the_sent_body_when_drop_params_meets_an_auto_added_beta( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + thinking: Final = {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}} + messages: Final = [{"role": "user", "content": "Say hi in French"}] + predicted: Final = prepare_native_baseline_body( + {"model": "claude-opus-4-8", "messages": messages, "max_tokens": 64, "thinking": thinking}, "claude-opus-4-8" + ) + config: Final = AnthropicMessagesConfig() + headers, _ = config.validate_anthropic_messages_environment( + headers={}, + model="claude-opus-4-8", + messages=[dict(message) for message in messages], + optional_params={"max_tokens": 64, "thinking": dict(thinking)}, + litellm_params={}, + api_key="sk-ant-test", + ) + sent: Final = config.transform_anthropic_messages_request( + model="claude-opus-4-8", + messages=[dict(message) for message in messages], + anthropic_messages_optional_request_params={"max_tokens": 64, "thinking": dict(thinking)}, + litellm_params=GenericLiteLLMParams(), + headers=headers, + ) + assert predicted is not None + assert predicted["thinking"] == sent["thinking"] == thinking diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 547db7a8c21..403417709f0 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -1,7 +1,6 @@ import base64 import copy import json -import logging import os import re from collections.abc import Iterator, Mapping @@ -10883,3 +10882,17 @@ def test_converse_keeps_beta_gated_thinking_fields_the_body_anthropic_beta_carri "block_binding": {"prefix_mismatch_behavior": "error"}, "display": "updates", } + + +def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_betas( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") + fields: Final = _converse_bound_thinking_request( + {"anthropic-beta": "context-1m-2025-08-07"}, body_betas=("not-a-real-beta-2099",) + ) + assert sorted(fields["anthropic_beta"]) == [ + "context-1m-2025-08-07", + "thinking-binding-controls-2026-08-01", + "thinking-display-updates-2026-08-18", + ] diff --git a/tests/unit/llms/bedrock/test_claude_platform_provider.py b/tests/unit/llms/bedrock/test_claude_platform_provider.py index 2353a45139b..9142fe45752 100644 --- a/tests/unit/llms/bedrock/test_claude_platform_provider.py +++ b/tests/unit/llms/bedrock/test_claude_platform_provider.py @@ -207,6 +207,26 @@ def test_claude_platform_api_key_auth_sets_workspace_and_key_headers(): assert headers["anthropic-beta"] == "skills-2025-10-02" +def test_claude_platform_chat_sends_the_binding_beta_when_thinking_carries_block_binding(): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + headers: Final = BedrockClaudePlatformConfig().validate_environment( + api_key="fake-platform-key", + headers={}, + model="claude-opus-5-5", + messages=[{"role": "user", "content": "hello"}], + optional_params={ + "workspace_id": "wrkspc_test", + "thinking": {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}, + }, + litellm_params={}, + ) + + assert "thinking-binding-controls-2026-08-01" in headers["anthropic-beta"].split(",") + + def test_claude_platform_does_not_use_standard_anthropic_api_key(monkeypatch): from litellm.llms.bedrock.claude_platform.transformation import ( BedrockClaudePlatformConfig, diff --git a/tests/unit/llms/databricks/messages/test_databricks_messages_transformation.py b/tests/unit/llms/databricks/messages/test_databricks_messages_transformation.py index f3f74eac38a..ae54d0e7437 100644 --- a/tests/unit/llms/databricks/messages/test_databricks_messages_transformation.py +++ b/tests/unit/llms/databricks/messages/test_databricks_messages_transformation.py @@ -324,9 +324,8 @@ async def _workspace_request_body(**request_params: object) -> dict[str, object] @pytest.mark.asyncio @pytest.mark.parametrize("drop_params", [True, False]) async def test_bound_thinking_with_display_updates_reaches_the_workspace_with_both_betas( - drop_params: bool, monkeypatch: pytest.MonkeyPatch + drop_params: bool, monkeypatch: pytest.MonkeyPatch, local_beta_headers_config: None ) -> None: - monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") monkeypatch.setattr(litellm, "drop_params", False) thinking: Final = { "type": "enabled", diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 6c10d524531..47c87c7e850 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -208,7 +208,7 @@ def test_no_safeguards_leaves_dangerous_tool_use_beta_header_out(): assert "dangerous-tool-use-2026-09-03" not in updated_headers.get("anthropic-beta", "") -def _validate_vertex_headers(client_headers, messages): +def _validate_vertex_headers(client_headers, messages, thinking: Mapping[str, object] | None = None): config = VertexAIPartnerModelsAnthropicMessagesConfig() litellm_params = { "vertex_ai_project": "test-project", @@ -224,7 +224,7 @@ def _validate_vertex_headers(client_headers, messages): headers=client_headers, model="claude-opus-5-5", messages=messages, - optional_params={"max_tokens": 64}, + optional_params={"max_tokens": 64, **({"thinking": thinking} if thinking is not None else {})}, litellm_params=litellm_params, api_base=None, ) @@ -256,6 +256,25 @@ def test_per_message_output_config_reaches_vertex_with_per_turn_control_beta(cli assert filtered["anthropic-beta"].split(",").count("per-turn-control-2026-07-01") == 1 +def test_block_binding_reaches_vertex_messages_with_its_beta_under_drop_params(local_beta_headers_config: None) -> None: + from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta + + thinking: Final = {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}} + messages: Final = [{"role": "user", "content": "Say hi in French"}] + headers: Final = update_headers_with_filtered_beta( + headers=_validate_vertex_headers({}, messages, thinking=thinking), provider="vertex_ai" + ) + request: Final = VertexAIPartnerModelsAnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-opus-5-5", + messages=[dict(message) for message in messages], + anthropic_messages_optional_request_params={"max_tokens": 64, "thinking": dict(thinking)}, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers=headers, + ) + assert "thinking-binding-controls-2026-08-01" in headers["anthropic-beta"].split(",") + assert request["thinking"] == thinking + + def test_no_per_message_output_config_leaves_per_turn_control_beta_out(): headers = _validate_vertex_headers({}, [{"role": "user", "content": "Hello"}]) diff --git a/tests/unit/test_anthropic_beta_gated_fields.py b/tests/unit/test_anthropic_beta_gated_fields.py index 2f1dcf3e854..f1edde64396 100644 --- a/tests/unit/test_anthropic_beta_gated_fields.py +++ b/tests/unit/test_anthropic_beta_gated_fields.py @@ -119,7 +119,9 @@ def test_system_message_loses_exactly_the_fields_whose_betas_are_absent() -> Non def test_per_turn_control_beta_also_unlocks_system_output_config() -> None: messages: Final = [{"role": "system", "output_config": {"effort": "low"}, "content": "terse"}] assert ( - strip_beta_gated_system_messages(messages, frozenset({"per-turn-control-2026-07-01"}), drop_params=True).messages + strip_beta_gated_system_messages( + messages, frozenset({"per-turn-control-2026-07-01"}), drop_params=True + ).messages is messages ) assert ( @@ -135,7 +137,10 @@ def test_per_turn_control_beta_also_unlocks_system_output_config() -> None: def test_inline_tools_beta_also_unlocks_system_tool_changes() -> None: messages: Final = [ - {"role": "system", "content": [{"type": "tool_removal", "tool_names": ["lookup"]}, {"type": "text", "text": "x"}]} + { + "role": "system", + "content": [{"type": "tool_removal", "tool_names": ["lookup"]}, {"type": "text", "text": "x"}], + } ] assert ( strip_beta_gated_system_messages( @@ -168,6 +173,28 @@ def test_system_message_emptied_by_the_strip_is_dropped() -> None: ) +def test_emptied_trailing_system_turn_after_an_assistant_turn_is_sent_as_is() -> None: + messages: Final = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "ok"}, + {"role": "system", "content": [{"type": "tool_removal", "tool_names": ["lookup"]}]}, + ] + result: Final = strip_beta_gated_system_messages(messages, frozenset(), drop_params=True) + assert result.messages is messages + assert result.stripped == () + + +def test_emptied_system_turn_is_dropped_when_a_user_turn_still_ends_the_conversation() -> None: + messages: Final = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "ok"}, + {"role": "system", "content": [{"type": "tool_removal", "tool_names": ["lookup"]}]}, + {"role": "user", "content": "go on"}, + ] + result: Final = strip_beta_gated_system_messages(messages, frozenset(), drop_params=True) + assert [message["role"] for message in result.messages] == ["user", "assistant", "user"] + + def test_system_turn_emptied_by_the_strip_stays_as_a_directive_when_its_output_config_survives() -> None: directive: Final = { "role": "system", @@ -222,7 +249,9 @@ def test_request_strip_is_identity_when_nothing_is_gated() -> None: @pytest.mark.parametrize("drop_params", [None, False, "false"]) -def test_without_drop_params_every_gated_field_is_sent_as_is(drop_params: object, monkeypatch: pytest.MonkeyPatch) -> None: +def test_without_drop_params_every_gated_field_is_sent_as_is( + drop_params: object, monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.setattr(litellm, "drop_params", False) messages: Final = [{"role": "user", "content": "hi"}, _system_turn()] request: Final = {"thinking": BOUND_THINKING, "messages": messages} @@ -259,5 +288,7 @@ def test_every_strip_is_logged_once_at_warning_with_the_fields_model_and_provide log_beta_gated_strips((), model="claude", provider="azure_ai") warnings: Final = [record for record in caplog.records if record.levelno == logging.WARNING] assert len(warnings) == 1 - assert "['thinking.block_binding']" in warnings[0].getMessage() - assert "model=claude provider=azure_ai" in warnings[0].getMessage() + assert ( + "Dropped thinking.block_binding (unlocked by anthropic-beta thinking-binding-controls-2026-08-01) for " + "model=claude provider=azure_ai" in warnings[0].getMessage() + )