scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-07 07:58:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 150

fix(anthropic): send the thinking betas on every leg and keep fields where the beta is unknown

Browse source 
Vertex messages and Claude Platform chat now add the thinking betas when thinking carries
block_binding or display updates. Converse sends only the caller's header betas when it has them.
A provider with no row in the beta config forwards beta-gated fields instead of dropping them,
an emptied trailing system turn that would leave an assistant prefill is sent as is, and the
prompt cache baseline body matches the body that goes out. The drop WARNING now names the beta
that unlocks each field
This commit is contained in:
mateo-berri 2026-10-28 17:00:27 -07:00
parent b42590dc20
commit 39b9b8d787
15 changed files with 456 additions and 153 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
litellm/anthropic_beta_gated_fields.py
View file

@ -196,11 +196,17 @@ def _strip_beta_gated_message(message: object, active_betas: Set[str]) -> Stripp
return _strip_beta_gated_system_message(mapping, active_betas)
def _ends_on_assistant(messages: Sequence[object]) -> bool:
last: Final = _as_mapping(messages[-1]) if messages else None
return last is not None and last.get("role") == "assistant"
def strip_beta_gated_system_messages(
messages: Sequence[object], active_betas: Set[str], *, drop_params: object
) -> StrippedMessages:
"""Drop the gated fields of every mid-conversation ``role: system`` message. Returns the same
sequence when nothing is gated."""
sequence when nothing is gated, or when dropping an emptied system turn would leave the conversation
ending on an assistant turn, which the model would read as a prefill to continue."""
if not drops_unsupported_params(drop_params):
return StrippedMessages(messages=messages, stripped=())
per_message: Final = tuple(_strip_beta_gated_message(message, active_betas) for message in messages)
@ -209,12 +215,12 @@ def strip_beta_gated_system_messages(
) # comprehension-ok: flattening per-message strip reports
if not stripped:
return StrippedMessages(messages=messages, stripped=())
return StrippedMessages(
messages=[
message for result in per_message for message in result.messages
], # comprehension-ok: flattening per-message results
stripped=stripped,
)
kept: Final = [
message for result in per_message for message in result.messages
] # comprehension-ok: flattening per-message results
if _ends_on_assistant(kept) and not _ends_on_assistant(messages):
return StrippedMessages(messages=messages, stripped=())
return StrippedMessages(messages=kept, stripped=stripped)
def strip_beta_gated_request_fields(
@ -291,12 +297,18 @@ def strip_beta_gated_native_request(
def log_beta_gated_strips(stripped: Sequence[str], model: str | None, provider: str) -> None:
if not stripped:
return
described: Final = ", ".join(
f"{field} (unlocked by anthropic-beta {' or '.join(sorted(_FIELD_BETAS[field]))})"
if field in _FIELD_BETAS
else field
for field in sorted(set(stripped))
)
verbose_logger.warning(
"Dropped %s for model=%s provider=%s because drop_params is on and the provider would reject them on "
"this request, either since this provider leg does not send the anthropic-beta that unlocks them (see "
"this request, either since this provider leg does not send that anthropic-beta (see "
"anthropic_beta_headers_config.json) or since the model turns mid-conversation system messages into user "
"messages.",
sorted(set(stripped)),
described,
model,
provider,
)

4
litellm/anthropic_beta_headers_manager.py
View file

@ -241,6 +241,10 @@ def get_provider_name(provider: str) -> str:
return aliases.get(provider, provider)
def has_beta_headers_config(provider: str) -> bool:
return get_provider_name(provider) in _load_beta_headers_config()
def filter_and_transform_beta_headers(
beta_headers: list[str],
provider: str,

21
litellm/llms/anthropic/messages/native/transformation.py
View file

@ -12,7 +12,11 @@ from litellm.anthropic_beta_gated_fields import (
strip_beta_gated_system_messages,
unstripped_native_request,
)
from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config
from litellm.anthropic_beta_headers_manager import (
get_unsupported_headers,
has_beta_headers_config,
is_beta_header_supported_by_bundled_config,
)
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -230,8 +234,10 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
self, messages: Sequence[object], model: str, drop_params: object
) -> tuple[Mapping[str, object], ...]:
validated: Final = _MESSAGE_MAPPINGS.validate_python(messages)
if not has_beta_headers_config(self._resolved_provider):
return validated
gated: Final = strip_beta_gated_system_messages(validated, frozenset(), drop_params=drop_params)
log_beta_gated_strips(gated.stripped, model=model, provider=self.custom_llm_provider or "anthropic")
log_beta_gated_strips(gated.stripped, model=model, provider=self._resolved_provider)
return _MESSAGE_MAPPINGS.validate_python(gated.messages) if gated.stripped else validated
def _normalize_system_role_messages(
@ -351,7 +357,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
model=model,
),
)
return self._finalize_messages_headers(headers, optional_params, messages), api_base
return self.finalize_messages_headers(headers, optional_params, messages), api_base
async def avalidate_anthropic_messages_environment(
self,
@ -391,7 +397,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
model=model,
),
)
return self._finalize_messages_headers(oauth_headers, optional_params, messages), api_base
return self.finalize_messages_headers(oauth_headers, optional_params, messages), api_base
def _require_auth_header(self, auth_header: Mapping[str, str] | None, model: str) -> Mapping[str, str]:
if auth_header is None:
@ -425,7 +431,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
return config_allows_workload_identity(self)
def _finalize_messages_headers(
def finalize_messages_headers(
self,
headers: dict, # mutable-ok: out-param
optional_params: dict, # mutable-ok: out-param
@ -636,8 +642,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
def strips_beta_gated_fields_from_final_headers(self) -> bool:
"""False on legs whose final ``anthropic_beta`` list is only known after this transform (Bedrock Invoke),
which strip beta-gated fields themselves once that list exists."""
return True
which strip beta-gated fields themselves once that list exists, and on providers with no row in
``anthropic_beta_headers_config.json``, whose filter drops every beta and so cannot say which fields fail."""
return has_beta_headers_config(self._resolved_provider)
def transform_anthropic_messages_request(
self,

5
litellm/llms/anthropic/prompt_cache_prediction.py
View file

@ -661,12 +661,13 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M
TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()),
dict(options),
)
body: Final = AnthropicMessagesConfig().transform_anthropic_messages_request(
config: Final = AnthropicMessagesConfig()
body: Final = config.transform_anthropic_messages_request(
model=resolved_model,
messages=strip_provider_specific_fields_from_anthropic_messages(messages),
anthropic_messages_optional_request_params=filtered,
litellm_params=GenericLiteLLMParams(),
headers={},
headers=config.finalize_messages_headers({}, dict(filtered), list(messages)), # pyright: ignore[reportUnknownMemberType] # the shared helper takes and returns bare dicts
)
return MappingProxyType(_JSON_OBJECT.validate_python(body))
except Exception: # noqa: BLE001 # an unsupported hypothetical request is unavailable, never an inference failure

4
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -1830,14 +1830,12 @@ class AmazonConverseConfig(BaseConfig):
base_model: Final = BedrockModelInfo.get_base_model(model)
header_beta_list: Final = get_anthropic_beta_from_headers(headers or {})
body_beta_list: Final = sorted(
client_beta_list: Final = header_beta_list or sorted(
active_beta_values(
{},
additional_request_params.get("anthropic_beta"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # additional_request_params is a bare dict
)
- set(header_beta_list)
)
client_beta_list: Final = [*header_beta_list, *body_beta_list]
eager_beta: Final = (
(ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,)
if base_model.startswith("anthropic")

1
litellm/llms/bedrock/claude_platform/transformation.py
View file

@ -72,6 +72,7 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig):
),
code_execution_tool_used=self.is_code_execution_tool_used(tools=optional_params.get("tools")),
container_with_skills_used=self.is_container_with_skills_used(optional_params=optional_params),
thinking=optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict
)
anthropic_headers["anthropic-workspace-id"] = workspace_id
return {**headers, **anthropic_headers}

2
litellm/llms/databricks/messages/transformation.py
View file

@ -144,7 +144,7 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig)
("content-type" if key.lower() == "content-type" else key): value
for key, value in {**auth_headers, **client_anthropic_headers}.items()
}
finalized_headers: Final[dict[str, str]] = self._finalize_messages_headers( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared helper returns a bare dict # mutable-ok: becomes the outbound headers
finalized_headers: Final[dict[str, str]] = self.finalize_messages_headers( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the shared helper returns a bare dict # mutable-ok: becomes the outbound headers
headers=normalized_headers,
optional_params=optional_params,
messages=messages,

6
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py
View file

@ -174,6 +174,12 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
if messages_carry_output_config(messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
beta_values.update(
anthropic_model_info.thinking_betas(
optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict
)
)
if beta_values:
request_headers["anthropic-beta"] = ",".join(beta_values)

20
tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py
View file

@ -501,3 +501,23 @@ def test_messages_request_drops_clear_at_from_a_mid_conversation_system_turn_wit
)
assert request["messages"][1] == {"role": "system", "content": "be terse"}
assert system_turn == {"role": "system", "content": "be terse", "clear_at": "end_turn"}
def test_a_provider_without_a_beta_headers_row_forwards_beta_gated_fields_under_drop_params(
local_beta_headers_config: None,
) -> None:
thinking: Final = {"type": "enabled", "budget_tokens": 1024, "display": "updates"}
request: Final = MinimaxMessagesConfig().transform_anthropic_messages_request(
model="MiniMax-M2",
messages=[
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "Hello"},
{"role": "user", "content": "how are you"},
{"role": "system", "content": "Answer in French", "output_config": {"effort": "low"}},
],
anthropic_messages_optional_request_params={"max_tokens": 64, "thinking": dict(thinking)},
litellm_params=GenericLiteLLMParams(drop_params=True),
headers={},
)
assert request["thinking"] == thinking
assert request["messages"][-1]["output_config"] == {"effort": "low"}

414
tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py
View file

@ -13,7 +13,7 @@ import litellm
from litellm.caching.dual_cache import DualCache
from litellm.caching.llm_caching_handler import LLMClientCache
from litellm.llms.anthropic.count_tokens import handler as count_handler
from litellm.llms.anthropic.messages.native.transformation import DEFAULT_ANTHROPIC_API_VERSION
from litellm.llms.anthropic.messages.native.transformation import DEFAULT_ANTHROPIC_API_VERSION, AnthropicMessagesConfig
from litellm.llms.anthropic.prompt_cache_prediction import (
CountedPromptCachePlan,
NativePredictionTarget,
@ -26,6 +26,7 @@ from litellm.llms.anthropic.prompt_cache_prediction import (
parse_cache_plan,
parse_observed_cache,
parse_prompt,
prepare_native_baseline_body,
resolve_baseline_prediction_target,
resolve_prediction_target,
supported_prediction_headers,
@ -36,7 +37,7 @@ from litellm.proxy import proxy_server
from litellm.proxy.hooks.prompt_cache_prediction import PromptCacheObserver
from litellm.proxy.management_endpoints.prompt_cache_prediction import predict_arm
from litellm.proxy.utils import InternalUsageCache
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
from litellm.types.router import Deployment, GenericLiteLLMParams, LiteLLM_Params, ModelInfo
from litellm.types.utils import CacheCreationTokenDetails, ModelResponse, PromptTokensDetailsWrapper, Usage
_MODEL: Final = "claude-sonnet-5"
@ -50,9 +51,12 @@ def _body() -> dict[str, JsonValue]:
"model": _MODEL,
"system": "Keep this context",
"tools": [{"name": "lookup", "input_schema": {"type": "object"}}],
"messages": [{"role": "user", "content": [
{"type": "text", "text": "A cacheable prefix", "cache_control": {"type": "ephemeral"}}
]}],
"messages": [
{
"role": "user",
"content": [{"type": "text", "text": "A cacheable prefix", "cache_control": {"type": "ephemeral"}}],
}
],
}
@ -88,7 +92,8 @@ async def test_observer_records_only_version_supported_by_token_counter(version:
"httpx_response": httpx.Response(200, request=wire),
"first_api_call_start_time": datetime.fromtimestamp(1000.0),
"standard_logging_object": {
"status": "success", "model_id": _DEPLOYMENT,
"status": "success",
"model_id": _DEPLOYMENT,
"metadata": {"user_api_key_hash": _CALLER},
},
},
@ -104,13 +109,16 @@ async def test_observer_records_only_version_supported_by_token_counter(version:
assert await lookup(cache, other_scope, prefix, now=1010.0) is None
@pytest.mark.parametrize("headers, supported", [
({}, True),
({"Anthropic-Version": DEFAULT_ANTHROPIC_API_VERSION}, True),
({"anthropic-version": "2099-01-01"}, False),
({"Anthropic-Beta": ""}, False),
({"anthropic-beta": "future-feature"}, False),
])
@pytest.mark.parametrize(
"headers, supported",
[
({}, True),
({"Anthropic-Version": DEFAULT_ANTHROPIC_API_VERSION}, True),
({"anthropic-version": "2099-01-01"}, False),
({"Anthropic-Beta": ""}, False),
({"anthropic-beta": "future-feature"}, False),
],
)
def test_prediction_header_eligibility(headers: Mapping[str, str], supported: bool) -> None:
assert supported_prediction_headers(headers) is supported
@ -150,49 +158,89 @@ async def test_provider_count_uses_same_version_and_preserves_native_input(
@pytest.mark.parametrize("source", ["static", "database"])
@pytest.mark.asyncio
async def test_environment_credential_matches_native_count_and_observed_scope(
source: str, monkeypatch: pytest.MonkeyPatch,
source: str,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("LIT7658_PROVIDER_KEY", _KEY)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
params: Final = {
"model": f"anthropic/{_MODEL}", "api_key": "os.environ/LIT7658_PROVIDER_KEY",
"model": f"anthropic/{_MODEL}",
"api_key": "os.environ/LIT7658_PROVIDER_KEY",
"api_base": "https://api.anthropic.com",
}
router: Final = litellm.Router(model_list=[{
"model_name": "test-native", "litellm_params": dict(params), "model_info": {"id": _DEPLOYMENT},
}] if source == "static" else [], num_retries=0)
router: Final = litellm.Router(
model_list=[
{
"model_name": "test-native",
"litellm_params": dict(params),
"model_info": {"id": _DEPLOYMENT},
}
]
if source == "static"
else [],
num_retries=0,
)
if source == "database":
monkeypatch.setattr(proxy_server, "llm_router", router)
assert proxy_server.ProxyConfig()._add_deployment([SimpleNamespace(
model_id=_DEPLOYMENT, model_name="test-native", model_info={}, litellm_params=dict(params),
)]) == 1
assert (
proxy_server.ProxyConfig()._add_deployment(
[
SimpleNamespace(
model_id=_DEPLOYMENT,
model_name="test-native",
model_info={},
litellm_params=dict(params),
)
]
)
== 1
)
deployment: Final = router.get_deployment(_DEPLOYMENT)
assert deployment is not None
target: Final = resolve_prediction_target(deployment.litellm_params)
assert isinstance(target, NativePredictionTarget)
body: Final = _body()
with respx.mock() as upstream:
native: Final = upstream.post("https://api.anthropic.com/v1/messages").respond(200, json={
"id": "msg_test", "type": "message", "role": "assistant", "model": _MODEL,
"content": [{"type": "text", "text": "Hello"}], "stop_reason": "end_turn", "stop_sequence": None,
"usage": {"input_tokens": 11, "output_tokens": 1, "cache_read_input_tokens": 300},
})
native: Final = upstream.post("https://api.anthropic.com/v1/messages").respond(
200,
json={
"id": "msg_test",
"type": "message",
"role": "assistant",
"model": _MODEL,
"content": [{"type": "text", "text": "Hello"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 11, "output_tokens": 1, "cache_read_input_tokens": 300},
},
)
counter: Final = upstream.post("https://api.anthropic.com/v1/messages/count_tokens").respond(
200, json={"input_tokens": 311},
200,
json={"input_tokens": 311},
)
await router.aanthropic_messages(
model="test-native", max_tokens=1, **{key: value for key, value in body.items() if key != "model"},
model="test-native",
max_tokens=1,
**{key: value for key, value in body.items() if key != "model"},
)
assert await count_prompt_tokens(target.model, target.api_key, body) == 311
assert native.call_count == counter.call_count == 1
assert native.calls.last.request.headers["x-api-key"] == counter.calls.last.request.headers["x-api-key"] == _KEY
observed: Final = parse_observed_cache(native.calls.last.request, ModelResponse(
model=_MODEL, usage=Usage(
prompt_tokens=311, completion_tokens=1, total_tokens=312,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300),
observed: Final = parse_observed_cache(
native.calls.last.request,
ModelResponse(
model=_MODEL,
usage=Usage(
prompt_tokens=311,
completion_tokens=1,
total_tokens=312,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300),
),
),
), _CALLER, _DEPLOYMENT)
_CALLER,
_DEPLOYMENT,
)
assert observed is not None
assert observed.scope == cache_scope(_CALLER, _DEPLOYMENT, target.api_key, target.model)
@ -200,15 +248,26 @@ async def test_environment_credential_matches_native_count_and_observed_scope(
@pytest.mark.parametrize("inline_key", [None, _KEY])
@pytest.mark.asyncio
async def test_named_credential_is_explicitly_unsupported_before_count(
inline_key: str | None, monkeypatch: pytest.MonkeyPatch,
inline_key: str | None,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "credential_list", [CredentialItem(
credential_name="test-named", credential_info={}, credential_values={"api_key": "test-named-provider-key"},
)])
monkeypatch.setattr(
litellm,
"credential_list",
[
CredentialItem(
credential_name="test-named",
credential_info={},
credential_values={"api_key": "test-named-provider-key"},
)
],
)
deployment: Final = Deployment(
model_name="test-native",
litellm_params=LiteLLM_Params(
model=f"anthropic/{_MODEL}", api_key=inline_key, litellm_credential_name="test-named",
model=f"anthropic/{_MODEL}",
api_key=inline_key,
litellm_credential_name="test-named",
),
model_info=ModelInfo(id=_DEPLOYMENT),
)
@ -232,8 +291,7 @@ def _cache_plan(body: Mapping[str, JsonValue]) -> PromptCachePlan:
def _text(text: str, ttl: str | None = None) -> dict[str, JsonValue]:
return {"type": "text", "text": text,
**({"cache_control": {"type": "ephemeral", "ttl": ttl}} if ttl else {})}
return {"type": "text", "text": text, **({"cache_control": {"type": "ephemeral", "ttl": ttl}} if ttl else {})}
def _prompt(*blocks: dict[str, JsonValue], role: str = "user", **options: JsonValue) -> dict[str, JsonValue]:
@ -251,16 +309,19 @@ def test_public_predictor_preserves_string_message_policy(text: str, supported:
def test_cache_plan_preserves_hierarchical_prefixes_and_public_policy() -> None:
body: Final = _prompt(
_text("First turn", "5m"), system=[_text("Stable instructions", "1h")],
tools=[{"name": "lookup", "input_schema": {"type": "object"},
"cache_control": {"type": "ephemeral", "ttl": "1h"}}],
_text("First turn", "5m"),
system=[_text("Stable instructions", "1h")],
tools=[
{"name": "lookup", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral", "ttl": "1h"}}
],
)
plan: Final = _cache_plan(body)
changed: Final = _cache_plan({**body, "system": [_text("Changed instructions", "1h")]})
assert tuple(marker.ttl_seconds for marker in plan.breakpoints) == (3600, 3600, 300)
assert plan.breakpoints[0].fingerprint == changed.breakpoints[0].fingerprint
assert all(left.fingerprint != right.fingerprint for left, right
in zip(plan.breakpoints[1:], changed.breakpoints[1:]))
assert all(
left.fingerprint != right.fingerprint for left, right in zip(plan.breakpoints[1:], changed.breakpoints[1:])
)
assert plan.breakpoints[0].prefix_body == {
"tools": [{"name": "lookup", "input_schema": {"type": "object"}}],
"messages": [],
@ -268,51 +329,80 @@ def test_cache_plan_preserves_hierarchical_prefixes_and_public_policy() -> None:
assert parse_prompt(body) is None
@pytest.mark.parametrize("kind, added, matches", [
("text", 19, True), ("text", 20, False), ("tool_use", 30, True),
("tool_result", 30, True),
])
@pytest.mark.parametrize(
"kind, added, matches",
[
("text", 19, True),
("text", 20, False),
("tool_use", 30, True),
("tool_result", 30, True),
],
)
def test_cache_plan_lookback_counts_native_positions(
kind: str, added: int, matches: bool,
kind: str,
added: int,
matches: bool,
) -> None:
previous: Final = _cache_plan(_body())
appended: Final[list[dict[str, JsonValue]]] = [
{"type": "tool_use", "id": f"tool_{index}", "name": "lookup", "input": {}}
if kind == "tool_use" else
{"type": "tool_result", "tool_use_id": f"tool_{index}", "content": "done"}
if kind == "tool_result" else
{"type": "text", "text": f"Added {index}"}
if kind == "tool_use"
else {"type": "tool_result", "tool_use_id": f"tool_{index}", "content": "done"}
if kind == "tool_result"
else {"type": "text", "text": f"Added {index}"}
for index in range(added)
]
current: Final = _cache_plan({**_body(), **_prompt(
_text("A cacheable prefix"), *appended[:-1],
{**appended[-1], "cache_control": {"type": "ephemeral"}},
)})
assert (previous.breakpoints[0].fingerprint
in current.breakpoints[0].lookback_fingerprints) is matches
current: Final = _cache_plan(
{
**_body(),
**_prompt(
_text("A cacheable prefix"),
*appended[:-1],
{**appended[-1], "cache_control": {"type": "ephemeral"}},
),
}
)
assert (previous.breakpoints[0].fingerprint in current.breakpoints[0].lookback_fingerprints) is matches
@pytest.mark.parametrize("change, same_prefix, same_content", [
("tool_order", False, False), ("effort", False, False),
("standard_speed", True, True), ("ttl", False, True),
])
@pytest.mark.parametrize(
"change, same_prefix, same_content",
[
("tool_order", False, False),
("effort", False, False),
("standard_speed", True, True),
("ttl", False, True),
],
)
def test_cache_plan_identity_respects_settings_and_preserves_content(
change: str, same_prefix: bool, same_content: bool,
change: str,
same_prefix: bool,
same_content: bool,
) -> None:
tool_input: Final[dict[str, JsonValue]] = {"a": 1, "b": 2, "cache_control": {"ttl": "user-data"}}
block: Final[dict[str, JsonValue]] = {
"type": "tool_use", "id": "tool_1", "name": "lookup", "input": tool_input,
"type": "tool_use",
"id": "tool_1",
"name": "lookup",
"input": tool_input,
"cache_control": {"type": "ephemeral", "ttl": "5m"},
}
changed_block: Final = (
{**block, "input": dict(reversed(tool_input.items()))} if change == "tool_order" else
{**block, "cache_control": {"type": "ephemeral", "ttl": "1h"}} if change == "ttl" else block
{**block, "input": dict(reversed(tool_input.items()))}
if change == "tool_order"
else {**block, "cache_control": {"type": "ephemeral", "ttl": "1h"}}
if change == "ttl"
else block
)
before: Final = _cache_plan(_prompt(block, role="assistant", output_config={"effort": "low"})).breakpoints[0]
after: Final = _cache_plan(_prompt(
changed_block, role="assistant", output_config={"effort": "high" if change == "effort" else "low"},
**({"speed": "standard"} if change == "standard_speed" else {}),
)).breakpoints[0]
after: Final = _cache_plan(
_prompt(
changed_block,
role="assistant",
output_config={"effort": "high" if change == "effort" else "low"},
**({"speed": "standard"} if change == "standard_speed" else {}),
)
).breakpoints[0]
assert (before.fingerprint == after.fingerprint) is same_prefix
assert (before.fingerprint in after.lookback_fingerprints) is same_prefix
assert (before.content_fingerprint == after.content_fingerprint) is same_content
@ -323,30 +413,41 @@ def test_cache_plan_identity_respects_settings_and_preserves_content(
def test_cache_plan_automatic_cache_and_thinking_use_last_cacheable_block() -> None:
body: Final = _prompt(
_text("A stable answer"), {"type": "thinking", "thinking": "Thinking", "signature": "signature"},
role="assistant", thinking={"type": "adaptive"}, cache_control={"type": "ephemeral", "ttl": "1h"},
_text("A stable answer"),
{"type": "thinking", "thinking": "Thinking", "signature": "signature"},
role="assistant",
thinking={"type": "adaptive"},
cache_control={"type": "ephemeral", "ttl": "1h"},
)
plan: Final = _cache_plan(body)
assert len(plan.breakpoints) == 1
assert plan.breakpoints[0].ttl_seconds == 3600
assert plan.breakpoints[0].prefix_body == {
"thinking": {"type": "adaptive"},
"messages": [{"role": "assistant", "content": [
{"type": "text", "text": "A stable answer"},
]}],
"messages": [
{
"role": "assistant",
"content": [
{"type": "text", "text": "A stable answer"},
],
}
],
}
assert parse_prompt(body) is None
@pytest.mark.parametrize("body, reason", [
(_prompt({"type": "image"}), "unsupported_prompt_shape"),
({**_body(), "unknown_native_setting": True}, "unsupported_prompt_shape"),
({**_body(), "cache_control": {"type": "ephemeral", "ttl": "1h"}},
"conflicting_cache_ttl"),
(_prompt(_text("five", "5m"), _text("hour", "1h")), "invalid_cache_ttl_order"),
])
@pytest.mark.parametrize(
"body, reason",
[
(_prompt({"type": "image"}), "unsupported_prompt_shape"),
({**_body(), "unknown_native_setting": True}, "unsupported_prompt_shape"),
({**_body(), "cache_control": {"type": "ephemeral", "ttl": "1h"}}, "conflicting_cache_ttl"),
(_prompt(_text("five", "5m"), _text("hour", "1h")), "invalid_cache_ttl_order"),
],
)
def test_cache_plan_unsupported_is_explicit(
body: Mapping[str, JsonValue], reason: str,
body: Mapping[str, JsonValue],
reason: str,
) -> None:
result: Final = parse_cache_plan(body)
assert isinstance(result, UnsupportedCachePlan)
@ -354,30 +455,46 @@ def test_cache_plan_unsupported_is_explicit(
@pytest.mark.asyncio
@pytest.mark.parametrize("model, counts, reason", [
(None, (100, 150, 200), None),
(None, (100, 201, 200), "inconsistent_prefix_token_count"),
(None, (151, 150, 200), "inconsistent_prefix_token_count"),
(None, (None, 150, 200), "token_count_unavailable"),
("claude-opus-5", (100, 150, 200), None),
("claude-sonnet-5", (100, 150, 200), None),
("declared-cache-model", (100, 150, 200), None),
("unknown-cache-model", (100, 150, 200), "unsupported_thinking_cache_semantics"),
("claude-haiku-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"),
("claude-sonnet-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"),
])
@pytest.mark.parametrize(
"model, counts, reason",
[
(None, (100, 150, 200), None),
(None, (100, 201, 200), "inconsistent_prefix_token_count"),
(None, (151, 150, 200), "inconsistent_prefix_token_count"),
(None, (None, 150, 200), "token_count_unavailable"),
("claude-opus-5", (100, 150, 200), None),
("claude-sonnet-5", (100, 150, 200), None),
("declared-cache-model", (100, 150, 200), None),
("unknown-cache-model", (100, 150, 200), "unsupported_thinking_cache_semantics"),
("claude-haiku-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"),
("claude-sonnet-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"),
],
)
async def test_cache_plan_count_conserves_total_and_rejects_unknown(
model: str | None, counts: tuple[int | None, int | None, int], reason: str | None,
model: str | None,
counts: tuple[int | None, int | None, int],
reason: str | None,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setitem(litellm.model_cost, "declared-cache-model", {
"litellm_provider": "anthropic", "mode": "chat", "supports_thinking_cache_preservation": True,
})
plan: Final = _cache_plan(_prompt(
{"type": "thinking", "thinking": "Retained thought", "signature": "signature"}
if model else _text("first", "5m"),
_text("second", "5m"), _text("uncached"), role="assistant" if model else "user",
))
monkeypatch.setitem(
litellm.model_cost,
"declared-cache-model",
{
"litellm_provider": "anthropic",
"mode": "chat",
"supports_thinking_cache_preservation": True,
},
)
plan: Final = _cache_plan(
_prompt(
{"type": "thinking", "thinking": "Retained thought", "signature": "signature"}
if model
else _text("first", "5m"),
_text("second", "5m"),
_text("uncached"),
role="assistant" if model else "user",
)
)
async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
assert reason != "unsupported_thinking_cache_semantics", "Unverified thinking retention must skip counting"
@ -399,12 +516,15 @@ async def test_cache_plan_count_conserves_total_and_rejects_unknown(
@pytest.mark.parametrize("section", ["system", "tools"])
@pytest.mark.parametrize("rejects_prefix", (False, True))
async def test_native_count_preserves_settings_and_requires_every_prefix(
section: str, rejects_prefix: bool, monkeypatch: pytest.MonkeyPatch,
section: str,
rejects_prefix: bool,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
params: Final = LiteLLM_Params(
model=f"anthropic/{_MODEL}", api_key=_KEY,
model=f"anthropic/{_MODEL}",
api_key=_KEY,
api_base="https://gateway.example/v1/messages",
)
target: Final = resolve_baseline_prediction_target(params)
@ -413,19 +533,38 @@ async def test_native_count_preserves_settings_and_requires_every_prefix(
assert not isinstance(resolve_prediction_target(params), NativePredictionTarget)
body: Final = _body()
marker: Final[dict[str, JsonValue]] = {"type": "ephemeral", "ttl": "1h"}
body[section] = ([_text("A cached system", "1h")] if section == "system" else [{
"name": "lookup", "input_schema": {"type": "object"}, "cache_control": marker,
}])
plan: Final = _cache_plan({**body, **_prompt(
_text("A later prefix", "5m"), _text("An uncached suffix"),
thinking={"type": "adaptive"}, tool_choice={"type": "auto"}, output_config={"effort": "high"},
)})
body[section] = (
[_text("A cached system", "1h")]
if section == "system"
else [
{
"name": "lookup",
"input_schema": {"type": "object"},
"cache_control": marker,
}
]
)
plan: Final = _cache_plan(
{
**body,
**_prompt(
_text("A later prefix", "5m"),
_text("An uncached suffix"),
thinking={"type": "adaptive"},
tool_choice={"type": "auto"},
output_config={"effort": "high"},
),
}
)
assert len(plan.breakpoints) == 2
assert plan.breakpoints[0].prefix_body["messages"] == []
async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
return await count_prompt_tokens(
model, api_key, {**body, "max_tokens": 100}, api_base=target.api_base,
model,
api_key,
{**body, "max_tokens": 100},
api_base=target.api_base,
)
with respx.mock(assert_all_called=False) as upstream:
@ -434,12 +573,16 @@ async def test_native_count_preserves_settings_and_requires_every_prefix(
upstream.post(endpoint, json={**body, "model": _MODEL}).respond(
400 if rejects_prefix and index == 1 else 200,
json={"detail": {"error": "messages parameter is required"}}
if rejects_prefix and index == 1 else {"input_tokens": tokens},
if rejects_prefix and index == 1
else {"input_tokens": tokens},
)
for index, (body, tokens) in enumerate(
(
(plan.full_body, 6000),
(plan.breakpoints[0].prefix_body, 5000),
(plan.breakpoints[1].prefix_body, 5800),
)
)
for index, (body, tokens) in enumerate((
(plan.full_body, 6000), (plan.breakpoints[0].prefix_body, 5000),
(plan.breakpoints[1].prefix_body, 5800),
))
)
unexpected: Final = upstream.post(endpoint).respond(200, json={"input_tokens": 1})
result: Final = await count_cache_plan(target.model, target.api_key, plan, count)
@ -452,3 +595,32 @@ async def test_native_count_preserves_settings_and_requires_every_prefix(
assert tuple(marker.prefix_tokens for marker in result.breakpoints) == (5000, 5800)
assert tuple(route.call_count for route in routes) == (1, 1, 1)
assert unexpected.call_count == 0
def test_native_baseline_body_matches_the_sent_body_when_drop_params_meets_an_auto_added_beta(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "drop_params", True)
thinking: Final = {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}
messages: Final = [{"role": "user", "content": "Say hi in French"}]
predicted: Final = prepare_native_baseline_body(
{"model": "claude-opus-4-8", "messages": messages, "max_tokens": 64, "thinking": thinking}, "claude-opus-4-8"
)
config: Final = AnthropicMessagesConfig()
headers, _ = config.validate_anthropic_messages_environment(
headers={},
model="claude-opus-4-8",
messages=[dict(message) for message in messages],
optional_params={"max_tokens": 64, "thinking": dict(thinking)},
litellm_params={},
api_key="sk-ant-test",
)
sent: Final = config.transform_anthropic_messages_request(
model="claude-opus-4-8",
messages=[dict(message) for message in messages],
anthropic_messages_optional_request_params={"max_tokens": 64, "thinking": dict(thinking)},
litellm_params=GenericLiteLLMParams(),
headers=headers,
)
assert predicted is not None
assert predicted["thinking"] == sent["thinking"] == thinking

15
tests/unit/llms/bedrock/chat/test_converse_transformation.py
View file

@ -1,7 +1,6 @@
import base64
import copy
import json
import logging
import os
import re
from collections.abc import Iterator, Mapping
@ -10883,3 +10882,17 @@ def test_converse_keeps_beta_gated_thinking_fields_the_body_anthropic_beta_carri
"block_binding": {"prefix_mismatch_behavior": "error"},
"display": "updates",
}
def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_betas(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true")
fields: Final = _converse_bound_thinking_request(
{"anthropic-beta": "context-1m-2025-08-07"}, body_betas=("not-a-real-beta-2099",)
)
assert sorted(fields["anthropic_beta"]) == [
"context-1m-2025-08-07",
"thinking-binding-controls-2026-08-01",
"thinking-display-updates-2026-08-18",
]

20
tests/unit/llms/bedrock/test_claude_platform_provider.py
View file

@ -207,6 +207,26 @@ def test_claude_platform_api_key_auth_sets_workspace_and_key_headers():
assert headers["anthropic-beta"] == "skills-2025-10-02"
def test_claude_platform_chat_sends_the_binding_beta_when_thinking_carries_block_binding():
from litellm.llms.bedrock.claude_platform.transformation import (
BedrockClaudePlatformConfig,
)
headers: Final = BedrockClaudePlatformConfig().validate_environment(
api_key="fake-platform-key",
headers={},
model="claude-opus-5-5",
messages=[{"role": "user", "content": "hello"}],
optional_params={
"workspace_id": "wrkspc_test",
"thinking": {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}},
},
litellm_params={},
)
assert "thinking-binding-controls-2026-08-01" in headers["anthropic-beta"].split(",")
def test_claude_platform_does_not_use_standard_anthropic_api_key(monkeypatch):
from litellm.llms.bedrock.claude_platform.transformation import (
BedrockClaudePlatformConfig,

3
tests/unit/llms/databricks/messages/test_databricks_messages_transformation.py
View file

@ -324,9 +324,8 @@ async def _workspace_request_body(**request_params: object) -> dict[str, object]
@pytest.mark.asyncio
@pytest.mark.parametrize("drop_params", [True, False])
async def test_bound_thinking_with_display_updates_reaches_the_workspace_with_both_betas(
drop_params: bool, monkeypatch: pytest.MonkeyPatch
drop_params: bool, monkeypatch: pytest.MonkeyPatch, local_beta_headers_config: None
) -> None:
monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true")
monkeypatch.setattr(litellm, "drop_params", False)
thinking: Final = {
"type": "enabled",

23
tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py
View file

@ -208,7 +208,7 @@ def test_no_safeguards_leaves_dangerous_tool_use_beta_header_out():
assert "dangerous-tool-use-2026-09-03" not in updated_headers.get("anthropic-beta", "")
def _validate_vertex_headers(client_headers, messages):
def _validate_vertex_headers(client_headers, messages, thinking: Mapping[str, object] | None = None):
config = VertexAIPartnerModelsAnthropicMessagesConfig()
litellm_params = {
"vertex_ai_project": "test-project",
@ -224,7 +224,7 @@ def _validate_vertex_headers(client_headers, messages):
headers=client_headers,
model="claude-opus-5-5",
messages=messages,
optional_params={"max_tokens": 64},
optional_params={"max_tokens": 64, **({"thinking": thinking} if thinking is not None else {})},
litellm_params=litellm_params,
api_base=None,
)
@ -256,6 +256,25 @@ def test_per_message_output_config_reaches_vertex_with_per_turn_control_beta(cli
assert filtered["anthropic-beta"].split(",").count("per-turn-control-2026-07-01") == 1
def test_block_binding_reaches_vertex_messages_with_its_beta_under_drop_params(local_beta_headers_config: None) -> None:
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
thinking: Final = {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}
messages: Final = [{"role": "user", "content": "Say hi in French"}]
headers: Final = update_headers_with_filtered_beta(
headers=_validate_vertex_headers({}, messages, thinking=thinking), provider="vertex_ai"
)
request: Final = VertexAIPartnerModelsAnthropicMessagesConfig().transform_anthropic_messages_request(
model="claude-opus-5-5",
messages=[dict(message) for message in messages],
anthropic_messages_optional_request_params={"max_tokens": 64, "thinking": dict(thinking)},
litellm_params=GenericLiteLLMParams(drop_params=True),
headers=headers,
)
assert "thinking-binding-controls-2026-08-01" in headers["anthropic-beta"].split(",")
assert request["thinking"] == thinking
def test_no_per_message_output_config_leaves_per_turn_control_beta_out():
headers = _validate_vertex_headers({}, [{"role": "user", "content": "Hello"}])

41
tests/unit/test_anthropic_beta_gated_fields.py
View file

@ -119,7 +119,9 @@ def test_system_message_loses_exactly_the_fields_whose_betas_are_absent() -> Non
def test_per_turn_control_beta_also_unlocks_system_output_config() -> None:
messages: Final = [{"role": "system", "output_config": {"effort": "low"}, "content": "terse"}]
assert (
strip_beta_gated_system_messages(messages, frozenset({"per-turn-control-2026-07-01"}), drop_params=True).messages
strip_beta_gated_system_messages(
messages, frozenset({"per-turn-control-2026-07-01"}), drop_params=True
).messages
is messages
)
assert (
@ -135,7 +137,10 @@ def test_per_turn_control_beta_also_unlocks_system_output_config() -> None:
def test_inline_tools_beta_also_unlocks_system_tool_changes() -> None:
messages: Final = [
{"role": "system", "content": [{"type": "tool_removal", "tool_names": ["lookup"]}, {"type": "text", "text": "x"}]}
{
"role": "system",
"content": [{"type": "tool_removal", "tool_names": ["lookup"]}, {"type": "text", "text": "x"}],
}
]
assert (
strip_beta_gated_system_messages(
@ -168,6 +173,28 @@ def test_system_message_emptied_by_the_strip_is_dropped() -> None:
)
def test_emptied_trailing_system_turn_after_an_assistant_turn_is_sent_as_is() -> None:
messages: Final = [
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "ok"},
{"role": "system", "content": [{"type": "tool_removal", "tool_names": ["lookup"]}]},
]
result: Final = strip_beta_gated_system_messages(messages, frozenset(), drop_params=True)
assert result.messages is messages
assert result.stripped == ()
def test_emptied_system_turn_is_dropped_when_a_user_turn_still_ends_the_conversation() -> None:
messages: Final = [
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "ok"},
{"role": "system", "content": [{"type": "tool_removal", "tool_names": ["lookup"]}]},
{"role": "user", "content": "go on"},
]
result: Final = strip_beta_gated_system_messages(messages, frozenset(), drop_params=True)
assert [message["role"] for message in result.messages] == ["user", "assistant", "user"]
def test_system_turn_emptied_by_the_strip_stays_as_a_directive_when_its_output_config_survives() -> None:
directive: Final = {
"role": "system",
@ -222,7 +249,9 @@ def test_request_strip_is_identity_when_nothing_is_gated() -> None:
@pytest.mark.parametrize("drop_params", [None, False, "false"])
def test_without_drop_params_every_gated_field_is_sent_as_is(drop_params: object, monkeypatch: pytest.MonkeyPatch) -> None:
def test_without_drop_params_every_gated_field_is_sent_as_is(
drop_params: object, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
messages: Final = [{"role": "user", "content": "hi"}, _system_turn()]
request: Final = {"thinking": BOUND_THINKING, "messages": messages}
@ -259,5 +288,7 @@ def test_every_strip_is_logged_once_at_warning_with_the_fields_model_and_provide
log_beta_gated_strips((), model="claude", provider="azure_ai")
warnings: Final = [record for record in caplog.records if record.levelno == logging.WARNING]
assert len(warnings) == 1
assert "['thinking.block_binding']" in warnings[0].getMessage()
assert "model=claude provider=azure_ai" in warnings[0].getMessage()
assert (
"Dropped thinking.block_binding (unlocked by anthropic-beta thinking-binding-controls-2026-08-01) for "
"model=claude provider=azure_ai" in warnings[0].getMessage()
)