diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 800a0939cec..2efc6cf567d 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -61,6 +61,7 @@ from litellm.llms.custom_httpx.http_handler import get_httpx_client, header_valu from litellm.router_utils.deployment_pricing import deployment_pricing_models from litellm.rust_bridge.tokenizer import get_encoding from litellm.types.llms.anthropic import ( + ANTHROPIC_UNRENDERED_BLOCK_TYPES, AnthropicContentParamSource, AnthropicContentParamSourceFileId, AnthropicContentParamSourceUrl, @@ -1703,6 +1704,8 @@ def _count_content_list( block_text = str(c.get(_TEXT_FIELD_BY_BLOCK_TYPE[c["type"]]) or "") if block_text: num_tokens += count_function(block_text) + elif c["type"] in ANTHROPIC_UNRENDERED_BLOCK_TYPES: + continue elif c["type"] == "image_url": image_url = c.get("image_url") num_tokens += _count_image_tokens(image_url, use_default_image_token_count) @@ -1752,7 +1755,7 @@ def _count_content_list( f"Invalid content item type: {content_type}. " f"Expected str or dict with 'type' field " f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, " - f"compaction, tool_reference, search_result)." + f"compaction, tool_reference, search_result, fallback)." ) return num_tokens except Exception as e: diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index e1f9a7d540f..72abc3a13e8 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -202,6 +202,15 @@ def carries_signed_compaction_block(messages: Sequence[object]) -> bool: return any(map(_is_signed_compaction_block, chain.from_iterable(map(_message_blocks, messages)))) +def _is_fallback_block(block: object) -> bool: + fields: Final = as_str_mapping(block) + return fields is not None and fields.get("type") == "fallback" + + +def carries_fallback_block(messages: Sequence[object]) -> bool: + return any(map(_is_fallback_block, chain.from_iterable(map(_message_blocks, messages)))) + + def requires_native_compaction_beta( custom_llm_provider: str, optional_params: Mapping[str, object], @@ -265,6 +274,7 @@ def count_tokens_feature_betas( ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, AnthropicModelInfo().is_mid_conversation_tool_change_used(messages), ), + (ANTHROPIC_BETA_HEADER_VALUES.SERVER_SIDE_FALLBACK_2026_06_01.value, carries_fallback_block(messages)), ) if used ) diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 1c8f0c206cd..1d96a19d4a8 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -27,6 +27,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.common_utils import ( + carries_fallback_block, carries_signed_compaction_block, context_management_betas, context_management_with_edits, @@ -436,15 +437,22 @@ class AnthropicCountTokensConfig: bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta merges with the token-counting beta instead of replacing it. A history that carries a signed compaction block also needs the compaction beta, or Anthropic rejects the block - as an unknown content type, and ``context_management`` edits need the betas inference sends.""" + as an unknown content type. A history that echoes a fallback block needs the fallback beta + the same way, and ``context_management`` edits need the betas inference sends.""" compaction_betas: Final = ( (ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,) if carries_signed_compaction_block(messages) else () ) + fallback_betas: Final = ( + (ANTHROPIC_BETA_HEADER_VALUES.SERVER_SIDE_FALLBACK_2026_06_01.value,) + if carries_fallback_block(messages) + else () + ) request_betas: Final = ( ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, *compaction_betas, + *fallback_betas, *context_management_betas(context_management), ) return { diff --git a/litellm/llms/anthropic/messages/convertible_blocks.py b/litellm/llms/anthropic/messages/convertible_blocks.py index 09d662da543..35e830dca2b 100644 --- a/litellm/llms/anthropic/messages/convertible_blocks.py +++ b/litellm/llms/anthropic/messages/convertible_blocks.py @@ -13,6 +13,7 @@ import litellm from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.factory import sanitize_anthropic_tool_use_id from litellm.llms.anthropic.messages.native.utils import normalize_native_messages +from litellm.types.llms.anthropic import ANTHROPIC_UNRENDERED_BLOCK_TYPES from litellm.types.utils import TokenCountMessagesFormat _CONVERTIBLE_BLOCK_TYPES: Final = MappingProxyType( @@ -57,6 +58,8 @@ def _block_label(block: JsonValue, convertible: frozenset[str]) -> str | None: block_type: Final = block.get("type") if not isinstance(block_type, str): return "a block without a type" + if block_type in ANTHROPIC_UNRENDERED_BLOCK_TYPES: + return None if block_type not in convertible: return block_type return _source_label(block_type, block.get("source")) if block_type in _SOURCED_BLOCK_TYPES else None diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 152614e4385..eb0e5c57590 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -830,6 +830,9 @@ class ANTHROPIC_HOSTED_TOOLS(str, Enum): TOOL_SEARCH_TOOL = "tool_search_tool" +ANTHROPIC_UNRENDERED_BLOCK_TYPES: Final = frozenset({"fallback"}) + + class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): """ Known beta header values for Anthropic. @@ -847,6 +850,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): ADVISOR_TOOL_2026_03_01 = "advisor-tool-2026-03-01" PER_TURN_CONTROL_2026_07_01 = "per-turn-control-2026-07-01" DANGEROUS_TOOL_USE_2026_09_03 = "dangerous-tool-use-2026-09-03" + SERVER_SIDE_FALLBACK_2026_06_01 = "server-side-fallback-2026-06-01" # Tool search beta header constant (for Anthropic direct API and Microsoft Foundry) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 697ac76b4e5..4cef40399ff 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1982,6 +1982,23 @@ def test_token_counter_with_compaction_block(): assert token_counter(model=model, messages=unsigned) == token_counter(model=model, messages=as_text) +@pytest.mark.parametrize("model", ("anthropic/claude-sonnet-5-5", "gpt-5.4-mini", "deepseek/deepseek-v4-flash")) +def test_token_counter_counts_nothing_for_a_fallback_block(model: str) -> None: + user_turn = {"role": "user", "content": "hi"} + reply = {"type": "text", "text": "Hello"} + fallback_block = { + "type": "fallback", + "from": {"model": "first-model"}, + "to": {"model": "second-model"}, + "trigger": {"type": "refusal", "category": "cyber"}, + } + + with_block = [user_turn, {"role": "assistant", "content": [reply, fallback_block, reply]}] + without_block = [user_turn, {"role": "assistant", "content": [reply, reply]}] + + assert token_counter(model=model, messages=with_block) == token_counter(model=model, messages=without_block) + + def test_token_counter_counts_chat_compaction_blocks_like_their_summary_text(): """ Chat clients replay a compaction block in the assistant message's provider_specific_fields, and inference diff --git a/tests/unit/llms/anthropic/messages/test_convertible_blocks.py b/tests/unit/llms/anthropic/messages/test_convertible_blocks.py index 5fb6cc32dfb..e1ff64c1f9e 100644 --- a/tests/unit/llms/anthropic/messages/test_convertible_blocks.py +++ b/tests/unit/llms/anthropic/messages/test_convertible_blocks.py @@ -79,6 +79,14 @@ def test_blocks_both_conversions_drop_are_named(role: str, block: Mapping[str, o assert unconvertible_anthropic_blocks([{"role": role, "content": [block]}]) == (label,) +@pytest.mark.parametrize("role", ("user", "assistant")) +def test_a_fallback_block_is_accepted_because_it_renders_nothing_to_lose(role: str) -> None: + block: Final = {"type": "fallback", "from": {"model": "first"}, "to": {"model": "second"}} + + assert _converted_by_both_adapters(role, block) == (False, False) + assert unconvertible_anthropic_blocks([{"role": role, "content": [block]}]) == () + + def test_labels_are_listed_once_in_order_and_string_content_is_accepted() -> None: messages: Final = [ {"role": "user", "content": "plain text"}, diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index aa7fa2727b4..c0f2ceacfd1 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -28,9 +28,21 @@ from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( ) from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION +from litellm.llms.anthropic.common_utils import count_tokens_feature_betas from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES COMPACTION_BETA = ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value +FALLBACK_BETA = ANTHROPIC_BETA_HEADER_VALUES.SERVER_SIDE_FALLBACK_2026_06_01.value +FALLBACK_HISTORY = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Hello"}, + {"type": "fallback", "from": {"model": "first-model"}, "to": {"model": "second-model"}}, + ], + }, +] COMPACTED_HISTORY = [ { "role": "assistant", @@ -921,6 +933,24 @@ def test_headers_add_the_compaction_beta_only_for_a_compacted_history(): assert plain == ANTHROPIC_TOKEN_COUNTING_BETA_VERSION +def test_headers_add_the_fallback_beta_only_for_a_history_that_echoes_the_block(): + config = AnthropicCountTokensConfig() + auth_header = {"x-api-key": "sk-ant-api03-test-key"} + + with_block = config.get_count_tokens_headers(auth_header, FALLBACK_HISTORY)["anthropic-beta"].split(",") + plain = config.get_count_tokens_headers(auth_header, [{"role": "user", "content": "hi"}])["anthropic-beta"] + + assert FALLBACK_BETA in with_block + assert ANTHROPIC_TOKEN_COUNTING_BETA_VERSION in with_block + assert plain == ANTHROPIC_TOKEN_COUNTING_BETA_VERSION + + +@pytest.mark.parametrize("provider", ("azure_ai", "vertex_ai")) +def test_platform_count_betas_carry_the_fallback_beta_only_for_a_history_that_echoes_the_block(provider: str) -> None: + assert FALLBACK_BETA in count_tokens_feature_betas(provider, {}, FALLBACK_HISTORY) + assert FALLBACK_BETA not in count_tokens_feature_betas(provider, {}, [{"role": "user", "content": "hi"}]) + + @pytest.mark.asyncio async def test_handler_sends_the_compaction_beta_with_a_compacted_history(httpx_transport_clients): with respx.mock: