From 4b7b64574ae615d1e543b2430948ad8dc5ba0157 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:24:51 -0700 Subject: [PATCH 1/6] fix(count_tokens): count a compaction block instead of answering 500 A message history that replays Anthropic's signed compaction block made /v1/messages/count_tokens answer HTTP 500: neither the Anthropic nor the Vertex AI partner count-tokens request carried the compact-2026-09-04 beta, the provider rejected the block with a 400, the proxy fell back to the local tokenizer, and that raised on the unknown block type. Both provider count paths now add the compaction beta whenever the history carries a compaction block, and the local tokenizer counts the block's summary text like a text block, skipping the signature. --- litellm/litellm_core_utils/token_counter.py | 20 +++--- .../llms/anthropic/count_tokens/handler.py | 2 +- .../anthropic/count_tokens/transformation.py | 28 ++++++-- .../count_tokens/handler.py | 15 +++- .../litellm_core_utils/test_token_counter.py | 20 ++++++ ...t_anthropic_count_tokens_transformation.py | 51 ++++++++++++++ .../llms/anthropic/test_count_tokens_oauth.py | 2 +- .../test_count_tokens_compaction_beta.py | 69 +++++++++++++++++++ .../anthropic_endpoints/test_endpoints.py | 42 +++++++++++ 9 files changed, 232 insertions(+), 17 deletions(-) create mode 100644 tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_count_tokens_compaction_beta.py diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index b92eb74cbb1..24bf8c43fd6 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -5,6 +5,7 @@ import io import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from itertools import accumulate +from types import MappingProxyType from typing import Final, Literal, cast import anyio @@ -43,6 +44,7 @@ from litellm.types.llms.anthropic import ( AnthropicMessagesTextParam, AnthropicMessagesToolResultParam, AnthropicMessagesToolUseParam, + CompactionBlock, ) from litellm.types.llms.openai import ( AllMessageValues, @@ -898,6 +900,11 @@ def _count_anthropic_content( return tokens +_TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType( + {"text": "text", "thinking": "thinking", "redacted_thinking": "thinking", "compaction": "content"} +) + + def _count_content_list( count_function: TokenCounterFunction, content_list: str @@ -908,6 +915,7 @@ def _count_content_list( | AnthropicMessagesTextParam | AnthropicMessagesImageParam | AnthropicMessagesDocumentParam + | CompactionBlock ], use_default_image_token_count: bool, default_token_count: int | None, @@ -918,8 +926,8 @@ def _count_content_list( for c in content_list: if isinstance(c, str): num_tokens += count_function(c) - elif c["type"] == "text": - num_tokens += count_function(str(c.get("text", ""))) + elif c["type"] in _TEXT_FIELD_BY_BLOCK_TYPE: + num_tokens += count_function(str(c.get(_TEXT_FIELD_BY_BLOCK_TYPE[c["type"]]) or "")) elif c["type"] == "image_url": image_url = c.get("image_url") num_tokens += _count_image_tokens(image_url, use_default_image_token_count) @@ -949,12 +957,6 @@ def _count_content_list( use_default_image_token_count, default_token_count, ) - elif c["type"] in ("thinking", "redacted_thinking"): - # Claude extended thinking content block - # Count the thinking text and skip the opaque blobs (signature, redacted data) - thinking_text = str(c.get("thinking", "")) - if thinking_text: - num_tokens += count_function(thinking_text) elif c["type"] == "tool_reference": # Anthropic tool-search reference block: a lightweight pointer to # a deferred tool, e.g. {"type": "tool_reference", "tool_name": ...}. @@ -972,7 +974,7 @@ def _count_content_list( f"Invalid content item type: {content_type}. " f"Expected str or dict with 'type' field " f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, " - f"tool_reference)." + f"compaction, tool_reference)." ) return num_tokens except Exception as e: diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index b4f107cdef4..ca38db155c1 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -78,7 +78,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): verbose_logger.debug("Making request to: %s", endpoint_url) # Get required headers - headers: Final = self.get_count_tokens_headers(auth_header) + headers: Final = self.get_count_tokens_headers(auth_header, messages) # Use LiteLLM's async httpx client async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC) diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 1e97d56913a..b46887c2e78 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -13,11 +13,22 @@ from pydantic import JsonValue, TypeAdapter from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers from litellm.llms.anthropic.wif import resolve_anthropic_base +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES _COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue]) COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config") +def _content_carries_compaction_block(content: JsonValue) -> bool: + if not isinstance(content, list): + return False + return any(isinstance(block, dict) and block.get("type") == "compaction" for block in content) + + +def messages_carry_compaction_block(messages: Sequence[Mapping[str, JsonValue]]) -> bool: + return any(_content_carries_compaction_block(message.get("content")) for message in messages) + + class AnthropicCountTokensConfig: """ Configuration and transformation logic for Anthropic CountTokens API. @@ -73,18 +84,25 @@ class AnthropicCountTokensConfig: ) ) - def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]: + def get_count_tokens_headers( + self, auth_header: Mapping[str, str], messages: Sequence[Mapping[str, JsonValue]] + ) -> dict[str, str]: """The count-tokens headers around a resolved Anthropic auth header (``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta - merges with the token-counting beta instead of replacing it.""" + merges with the token-counting beta instead of replacing it. A history that carries a + signed compaction block also needs the compaction beta, or Anthropic rejects the block + as an unknown content type.""" + request_betas: Final = ( + (ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) + if messages_carry_compaction_block(messages) + else (ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,) + ) return { "Content-Type": "application/json", "anthropic-version": "2023-06-01", **auth_header, - "anthropic-beta": merge_anthropic_beta_headers( - auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION - ), + "anthropic-beta": merge_anthropic_beta_headers(auth_header.get("anthropic-beta"), request_betas), } def validate_request( diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 285380f5fc7..39e161c3341 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -8,9 +8,15 @@ their respective publisher-specific count-tokens endpoints. from typing import Any, Final +from pydantic import JsonValue, TypeAdapter + +from litellm.llms.anthropic.count_tokens.transformation import messages_carry_compaction_block from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES + +_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]]) class VertexAIPartnerModelsTokenCounter(VertexBase): @@ -164,7 +170,14 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): ) # Prepare headers - headers: Final = {"Authorization": f"Bearer {access_token}"} + headers: Final = { + "Authorization": f"Bearer {access_token}", + **( + {"anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value} + if messages_carry_compaction_block(_MESSAGES.validate_python(request_data["messages"])) + else {} + ), + } # Get async HTTP client from litellm import LlmProviders diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index e0c5c22d420..646567f7733 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1205,6 +1205,26 @@ def test_token_counter_with_redacted_thinking_content(): assert token_counter(model=model, messages=with_block) == token_counter(model=model, messages=without_block) + +def test_token_counter_with_compaction_block(): + """ + A replayed on-demand compaction block (Anthropic's signed summary of the earlier turns) counts its + summary text like a text block and nothing for its signature. It used to raise, which turned every + /v1/messages/count_tokens call on a compacted history into a 500 once the provider count fell back here. + """ + model = "anthropic/claude-sonnet-5-5" + summary = "The user is building a recipe app in Python and asked for one-sentence class descriptions." + follow_up = {"role": "user", "content": [{"type": "text", "text": "Now do the same for Ingredient."}]} + signed_block = {"type": "compaction", "content": summary, "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40} + + as_compaction = [{"role": "assistant", "content": [signed_block]}, follow_up] + unsigned = [{"role": "assistant", "content": [{"type": "compaction", "content": summary}]}, follow_up] + as_text = [{"role": "assistant", "content": [{"type": "text", "text": summary}]}, follow_up] + + assert token_counter(model=model, messages=as_compaction) == token_counter(model=model, messages=as_text) + assert token_counter(model=model, messages=unsigned) == token_counter(model=model, messages=as_text) + + def test_token_counter_with_tool_reference_block(): """ Regression test: a message containing an Anthropic tool-search diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 6a7ec13ec4c..5bce1be0abb 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -3,10 +3,27 @@ import pytest import respx import litellm +from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountTokensConfig, ) +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES + +COMPACTION_BETA = ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value +COMPACTED_HISTORY = [ + { + "role": "assistant", + "content": [ + { + "type": "compaction", + "content": "The user is building a recipe app and asked for one-sentence class descriptions.", + "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40, + } + ], + }, + {"role": "user", "content": "Now do the same for Ingredient."}, +] def test_transform_basic_request(): @@ -162,3 +179,37 @@ async def test_handler_posts_to_count_tokens_path_under_deployment_api_base(http assert route.called assert result == {"input_tokens": 7} + + +def test_headers_add_the_compaction_beta_only_for_a_compacted_history(): + config = AnthropicCountTokensConfig() + auth_header = {"x-api-key": "sk-ant-api03-test-key"} + + with_block = config.get_count_tokens_headers(auth_header, COMPACTED_HISTORY)["anthropic-beta"].split(",") + plain = config.get_count_tokens_headers(auth_header, [{"role": "user", "content": "hi"}])["anthropic-beta"] + + assert COMPACTION_BETA in with_block + assert ANTHROPIC_TOKEN_COUNTING_BETA_VERSION in with_block + assert plain == ANTHROPIC_TOKEN_COUNTING_BETA_VERSION + + +@pytest.mark.asyncio +async def test_handler_sends_the_compaction_beta_with_a_compacted_history(httpx_transport_clients): + """Anthropic validates the compaction block on count_tokens too: without the beta it answers 400 + (an unexpected content block type), the proxy falls back to the local tokenizer, and that used to + reject the block as well, so the whole call came back as a 500.""" + with respx.mock: + route = respx.post("https://gateway.example/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 404}) + ) + result = await AnthropicCountTokensHandler().handle_count_tokens_request( + model="claude-sonnet-5-5", + messages=COMPACTED_HISTORY, + auth_header={"x-api-key": "sk-ant-api03-test-key"}, + api_base="https://gateway.example", + ) + + assert result == {"input_tokens": 404} + betas = route.calls.last.request.headers["anthropic-beta"].split(",") + assert COMPACTION_BETA in betas + assert ANTHROPIC_TOKEN_COUNTING_BETA_VERSION in betas diff --git a/tests/unit/llms/anthropic/test_count_tokens_oauth.py b/tests/unit/llms/anthropic/test_count_tokens_oauth.py index 96d909a4b3f..9a137d055ec 100644 --- a/tests/unit/llms/anthropic/test_count_tokens_oauth.py +++ b/tests/unit/llms/anthropic/test_count_tokens_oauth.py @@ -42,7 +42,7 @@ FEDERATED_DEPLOYMENT = { def count_tokens_headers_for(api_key: str) -> dict[str, str]: auth_header = AnthropicModelInfo.get_auth_header(api_key=api_key) assert auth_header is not None - return AnthropicCountTokensConfig().get_count_tokens_headers(auth_header) + return AnthropicCountTokensConfig().get_count_tokens_headers(auth_header, [{"role": "user", "content": "hi"}]) @pytest.fixture diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_count_tokens_compaction_beta.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_count_tokens_compaction_beta.py new file mode 100644 index 00000000000..ae02ed3d866 --- /dev/null +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_count_tokens_compaction_beta.py @@ -0,0 +1,69 @@ +""" +A compacted history on the Vertex AI partner count-tokens endpoint needs the compaction beta: Vertex +validates the signed compaction block exactly like Anthropic and answers 400 without the header, which +the proxy then degrades into the local tokenizer. +""" + +import pytest + +import litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler as handler_mod +from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import ( + VertexAIPartnerModelsTokenCounter, +) +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES + +COMPACTION_BETA = ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value +COMPACTED_HISTORY = [ + { + "role": "assistant", + "content": [ + { + "type": "compaction", + "content": "The user is building a recipe app and asked for one-sentence class descriptions.", + "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40, + } + ], + }, + {"role": "user", "content": "Now do the same for Ingredient."}, +] +PLAIN_HISTORY = [{"role": "user", "content": "Now do the same for Ingredient."}] + + +@pytest.fixture +def posted_headers(monkeypatch): + captured = {} + + async def fake_ensure_access_token(self, credentials, project_id, custom_llm_provider): + return "fake-token", "fake-project" + + class FakeResponse: + status_code = 200 + + def json(self): + return {"input_tokens": 404} + + class FakeClient: + async def post(self, url, headers=None, json=None, **kwargs): + captured.update(headers or {}) + return FakeResponse() + + monkeypatch.setattr(VertexAIPartnerModelsTokenCounter, "_ensure_access_token_async", fake_ensure_access_token) + monkeypatch.setattr(handler_mod, "get_async_httpx_client", lambda **kwargs: FakeClient()) + return captured + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("messages", "beta_expected"), + [(COMPACTED_HISTORY, COMPACTION_BETA), (PLAIN_HISTORY, None)], +) +async def test_compaction_beta_is_sent_only_with_a_compacted_history(posted_headers, messages, beta_expected): + result = await VertexAIPartnerModelsTokenCounter().handle_count_tokens_request( + model="claude-sonnet-5-5", + request_data={"model": "claude-sonnet-5-5", "messages": messages}, + litellm_params={"vertex_location": "us-east5"}, + ) + + assert result["input_tokens"] == 404 + assert posted_headers["Authorization"] == "Bearer fake-token" + assert posted_headers.get("anthropic-beta") == beta_expected diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index 801f61aa498..5fcf29f2e09 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -452,6 +452,48 @@ class TestErrorLogCarriesCallId: assert call_id in record.getMessage() +class TestCountTokensCompactedHistory: + """A history carrying Anthropic's signed compaction block used to come back as a 500 from + /v1/messages/count_tokens once the count ran on the local tokenizer, which rejected the block type.""" + + @pytest.mark.asyncio + async def test_count_tokens_counts_a_compaction_block_like_its_summary_text(self): + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + summary = "The user is building a recipe app and asked for one-sentence class descriptions." + follow_up = {"role": "user", "content": "Now do the same for Ingredient."} + compacted = { + "model": "claude-sonnet-5-5", + "messages": [ + { + "role": "assistant", + "content": [{"type": "compaction", "content": summary, "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40}], + }, + follow_up, + ], + } + as_text = { + "model": "claude-sonnet-5-5", + "messages": [{"role": "assistant", "content": [{"type": "text", "text": summary}]}, follow_up], + } + request = MagicMock() + request.headers = {} + + async def count(body): + with ( + patch.object(ep, "_read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam + patch.object(proxy_server, "llm_router", None), # test-quality-ok: module global; with no router the count runs on the local tokenizer, the path that raised + ): + return await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + + counted = await count(compacted) + + assert counted["input_tokens"] > 0 + assert counted == await count(as_text) + + class TestEventLoggingBatchEndpoint: """Test the stubbed event logging batch endpoint""" From e9992f36e5e870499d4ee96b2f75e6523e77c936 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:42:18 -0700 Subject: [PATCH 2/6] fix(token_counter): skip the tokenizer for empty thinking and compaction text --- litellm/litellm_core_utils/token_counter.py | 4 +++- .../litellm_core_utils/test_token_counter.py | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 24bf8c43fd6..7de239c39c2 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -927,7 +927,9 @@ def _count_content_list( if isinstance(c, str): num_tokens += count_function(c) elif c["type"] in _TEXT_FIELD_BY_BLOCK_TYPE: - num_tokens += count_function(str(c.get(_TEXT_FIELD_BY_BLOCK_TYPE[c["type"]]) or "")) + block_text = str(c.get(_TEXT_FIELD_BY_BLOCK_TYPE[c["type"]]) or "") + if block_text: + num_tokens += count_function(block_text) elif c["type"] == "image_url": image_url = c.get("image_url") num_tokens += _count_image_tokens(image_url, use_default_image_token_count) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 646567f7733..43be15762a1 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1225,6 +1225,30 @@ def test_token_counter_with_compaction_block(): assert token_counter(model=model, messages=unsigned) == token_counter(model=model, messages=as_text) +def test_count_content_list_skips_the_tokenizer_for_empty_block_text(): + """ + An empty thinking block, a redacted_thinking block (no thinking text at all) and a compaction + block with no summary carry no tokens, even under a tokenizer that charges for the empty string + (the llama-2 tokenizer prepends its BOS token to every encode call). Counting them through the + tokenizer added one token per block on such models. + """ + from litellm.litellm_core_utils.token_counter import _count_content_list + + def charges_for_empty_text(text: str) -> int: + return len(text.split()) + 1 + + empty_blocks = [ + {"type": "thinking", "thinking": "", "signature": "EqcLCkYICxgCKkCrqu6lP..."}, + {"type": "thinking", "thinking": None, "signature": "EqcLCkYICxgCKkCrqu6lP..."}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy3va3pzix/LafPsn4aDFIT2Xlxh0L5L8rLVyIw"}, + {"type": "compaction", "content": ""}, + {"type": "text", "text": ""}, + ] + + assert _count_content_list(charges_for_empty_text, empty_blocks, False, None) == 0 + assert _count_content_list(charges_for_empty_text, [{"type": "thinking", "thinking": "two words"}], False, None) == 3 + + def test_token_counter_with_tool_reference_block(): """ Regression test: a message containing an Anthropic tool-search From 07c79b2c6b6449652b37fab4ecf81905c4ba5853 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:04:10 -0700 Subject: [PATCH 3/6] fix(count_tokens): forward context_management on Anthropic counts and count chat compaction blocks locally Vertex AI compaction support for inference and counting moves to its own PR, so the Vertex count handler is back to main here --- litellm/litellm_core_utils/token_counter.py | 19 +++++ litellm/llms/anthropic/common_utils.py | 20 +++++- .../llms/anthropic/count_tokens/handler.py | 4 +- .../anthropic/count_tokens/token_counter.py | 1 + .../anthropic/count_tokens/transformation.py | 31 +++++++-- .../messages/native/transformation.py | 24 +------ .../count_tokens/handler.py | 15 +--- .../litellm_core_utils/test_token_counter.py | 19 +++++ .../test_anthropic_messages_transformation.py | 28 +++++++- ...t_anthropic_count_tokens_transformation.py | 44 +++++++++++- .../vertex_ai/count_tokens/test_context.py | 3 +- .../test_count_tokens_compaction_beta.py | 69 ------------------- 12 files changed, 159 insertions(+), 118 deletions(-) delete mode 100644 tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_count_tokens_compaction_beta.py diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index de93f72dfc7..8256e3c59e2 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -15,6 +15,7 @@ import anyio import anyio.lowlevel import httpx import tiktoken +from pydantic import TypeAdapter from typing_extensions import ParamSpec, TypeVar import litellm @@ -842,6 +843,8 @@ def _count_messages( use_default_image_token_count, default_token_count, ) + elif key == "provider_specific_fields": + num_tokens += sum(map(params.count_function, _compaction_summaries(value))) elif key == "search_results" and isinstance(value, list): from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_search_results_text, @@ -856,6 +859,22 @@ def _count_messages( return num_tokens +_OBJECT_LIST: Final = TypeAdapter(list[object]) + + +def _compaction_summaries(provider_specific_fields: object) -> tuple[str, ...]: + fields: Final = as_str_mapping(provider_specific_fields) + blocks: Final = fields.get("compaction_blocks") if fields is not None else None + block_fields: Final = ( + tuple(map(as_str_mapping, _OBJECT_LIST.validate_python(blocks))) if isinstance(blocks, list) else () + ) + return tuple( + content + for block in block_fields + if block is not None and block.get("type") == "compaction" and isinstance(content := block.get("content"), str) + ) + + def _count_extra( count_function: TokenCounterFunction, tools: list[ChatCompletionToolParam] | None, diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index b10dd60380f..9380e5fc9c4 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -47,6 +47,7 @@ from litellm.llms.anthropic.wif import ( from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_HOSTED_TOOLS, ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER, ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, @@ -157,6 +158,23 @@ def requires_native_compaction_beta( ) +def context_management_betas(context_management: object) -> tuple[str, ...]: + spec: Final = as_str_mapping(context_management) + edits: Final = validated_object_list(spec.get("edits")) if spec is not None else None + edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits or ()) if edit is not None) + return tuple( + beta + for beta, used in ( + (ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, "compact_20260112" in edit_types), + ( + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + any(edit_type != "compact_20260112" for edit_type in edit_types), + ), + ) + if used + ) + + def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool: from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.utils import supports_prompt_caching @@ -1315,8 +1333,6 @@ class AnthropicModelInfo(BaseLLMModelInfo): if is_vertex_request is True: # Vertex AI requires web search beta header for web search to work if web_search_tool_used: - from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES - headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value elif len(all_betas) > 0: headers["anthropic-beta"] = ",".join(all_betas) diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index 58368f7d540..87912e50667 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + context_management: JsonValue = None, ) -> dict[str, JsonValue]: """ Handle a CountTokens request using httpx. @@ -69,6 +70,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools=tools, system=system, optional_params=optional_params, + context_management=context_management, ) verbose_logger.debug("Transformed request: %s", request_body) @@ -79,7 +81,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): verbose_logger.debug("Making request to: %s", endpoint_url) # Get required headers - headers: Final = self.get_count_tokens_headers(auth_header, messages) + headers: Final = self.get_count_tokens_headers(auth_header, messages, context_management) # Use LiteLLM's async httpx client async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC) diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 368bb929eeb..bffcb4b90fa 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -131,6 +131,7 @@ class AnthropicTokenCounter(BaseTokenCounter): tools=tools, system=system, optional_params=count_options, + context_management=context_management.spec if context_management is not None else None, ) if result is not None: diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 8bd9070dfca..2ec78f0bf0f 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -26,7 +26,11 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import split_leading_system_run, ) from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.anthropic.common_utils import carries_signed_compaction_block, merge_anthropic_beta_headers +from litellm.llms.anthropic.common_utils import ( + carries_signed_compaction_block, + context_management_betas, + merge_anthropic_beta_headers, +) from litellm.llms.anthropic.wif import resolve_anthropic_base from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes @@ -373,6 +377,7 @@ class AnthropicCountTokensConfig: tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + context_management: JsonValue = None, ) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries """ Transform request to Anthropic CountTokens format. @@ -392,7 +397,11 @@ class AnthropicCountTokensConfig: **MappingProxyType( { key: value - for key, value in (("system", counted_system), ("tools", tools)) + for key, value in ( + ("system", counted_system), + ("tools", tools), + ("context_management", context_management), + ) if value is not None } ), @@ -404,18 +413,26 @@ class AnthropicCountTokensConfig: ) def get_count_tokens_headers( - self, auth_header: Mapping[str, str], messages: Sequence[Mapping[str, JsonValue]] + self, + auth_header: Mapping[str, str], + messages: Sequence[Mapping[str, JsonValue]], + context_management: JsonValue = None, ) -> dict[str, str]: """The count-tokens headers around a resolved Anthropic auth header (``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta merges with the token-counting beta instead of replacing it. A history that carries a signed compaction block also needs the compaction beta, or Anthropic rejects the block - as an unknown content type.""" - request_betas: Final = ( - (ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) + as an unknown content type, and ``context_management`` edits need the betas inference sends.""" + compaction_betas: Final = ( + (ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,) if carries_signed_compaction_block(messages) - else (ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,) + else () + ) + request_betas: Final = ( + ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, + *compaction_betas, + *context_management_betas(context_management), ) return { "Content-Type": "application/json", diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index cc53858f842..c86a60270a6 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -31,6 +31,7 @@ from litellm.types.router import GenericLiteLLMParams from ...common_utils import ( AnthropicError, AnthropicModelInfo, + context_management_betas, merge_anthropic_beta_headers, optionally_handle_anthropic_oauth, requires_native_compaction_beta, @@ -792,28 +793,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if requires_native_compaction_beta(custom_llm_provider, optional_params, messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) - # Check for context management - context_management_param: Final = optional_params.get("context_management") - if context_management_param is not None: - # Check edits array for compact_20260112 type - edits: Final = context_management_param.get("edits", ()) - has_compact = False - has_other = False - - for edit in edits: - edit_type = edit.get("type", "") - if edit_type == "compact_20260112": - has_compact = True - else: - has_other = True - - # Add compact header if any compact edits exist - if has_compact: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) - - # Add context management header if any other edits exist - if has_other: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value) + beta_values.update(context_management_betas(optional_params.get("context_management"))) # Check for structured outputs. Anthropic's newer request shape nests # the schema under output_config.format; the older top-level diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 774f926c084..4be41e26326 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -8,16 +8,10 @@ their respective publisher-specific count-tokens endpoints. from typing import Any, Final -from pydantic import TypeAdapter - from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.anthropic.common_utils import carries_signed_compaction_block from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase -from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES - -_MESSAGES: Final = TypeAdapter(list[object]) _COUNT_TOKENS_PUBLISHERS: Final = ( ("claude", "anthropic"), @@ -163,14 +157,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): ) # Prepare headers - headers: Final = { - "Authorization": f"Bearer {access_token}", - **( - {"anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value} - if carries_signed_compaction_block(_MESSAGES.validate_python(request_data["messages"])) - else {} - ), - } + headers: Final = {"Authorization": f"Bearer {access_token}"} # Get async HTTP client from litellm import LlmProviders diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index b79031b0d9b..31c461803fe 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1651,6 +1651,25 @@ def test_token_counter_with_compaction_block(): assert token_counter(model=model, messages=unsigned) == token_counter(model=model, messages=as_text) +def test_token_counter_counts_chat_compaction_blocks_like_their_summary_text(): + """ + Chat clients replay a compaction block in the assistant message's provider_specific_fields, and inference + sends it to Anthropic as a content block, so the local count includes its summary text and skips its signature. + """ + model = "anthropic/claude-sonnet-5-5" + summary = "The user is building a recipe app in Python and asked for one-sentence class descriptions." + follow_up = {"role": "user", "content": "Now do the same for Ingredient."} + signed_block = {"type": "compaction", "content": summary, "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40} + + replayed = [ + {"role": "assistant", "content": None, "provider_specific_fields": {"compaction_blocks": [signed_block]}}, + follow_up, + ] + as_text = [{"role": "assistant", "content": summary}, follow_up] + + assert token_counter(model=model, messages=replayed) == token_counter(model=model, messages=as_text) + + def test_count_content_list_skips_the_tokenizer_for_empty_block_text(): """ An empty thinking block, a redacted_thinking block (no thinking text at all) and a compaction diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index aa6625561ba..693370c021d 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -17,7 +17,7 @@ from litellm.llms.anthropic.wif import get_anthropic_wif_token from litellm.llms.minimax.messages.transformation import MinimaxMessagesConfig from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig from litellm.llms.tencent.messages.transformation import TencentAnthropicMessagesConfig -from litellm.types.llms.anthropic import AnthropicFinishReason +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, AnthropicFinishReason from litellm.types.router import GenericLiteLLMParams from tests.unit.llms.anthropic.test_anthropic_wif import ( ScriptedPoster, @@ -236,6 +236,32 @@ def _derived_beta_header(config: AnthropicMessagesConfig) -> dict: ) +_COMPACT_EDIT_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value +_CLEARING_EDIT_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value + + +@pytest.mark.parametrize( + ("edits", "betas"), + ( + pytest.param([{"type": "compact_20260112"}], {_COMPACT_EDIT_BETA}, id="compact"), + pytest.param([{"type": "clear_tool_uses_20250919"}], {_CLEARING_EDIT_BETA}, id="clearing"), + pytest.param( + [{"type": "compact_20260112"}, {"type": "clear_thinking_20251015"}], + {_COMPACT_EDIT_BETA, _CLEARING_EDIT_BETA}, + id="both", + ), + ), +) +def test_context_management_edits_add_the_beta_each_edit_needs( + edits: list[dict[str, str]], betas: set[str], local_beta_headers_config: None +) -> None: + headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta( + headers={}, optional_params={"context_management": {"edits": edits}}, messages=[] + ) + + assert set(headers["anthropic-beta"].split(",")) == betas + + @pytest.mark.parametrize("config", _classifier_configs(), ids=type) def test_routes_whose_platform_runs_the_classifier_keep_safeguards_and_the_beta( config: AnthropicMessagesConfig, local_beta_headers_config: None diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 1642c0efffc..ca409ef2db2 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -26,7 +26,7 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, ) -from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse +from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES @@ -890,3 +890,45 @@ async def test_handler_sends_the_compaction_beta_with_a_compacted_history(httpx_ betas = route.calls.last.request.headers["anthropic-beta"].split(",") assert COMPACTION_BETA in betas assert ANTHROPIC_TOKEN_COUNTING_BETA_VERSION in betas + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@pytest.mark.parametrize( + ("edit_type", "sent_beta", "unsent_beta"), + ( + ( + "compact_20260112", + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + ), + ( + "clear_tool_uses_20250919", + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + ), + ), +) +async def test_counter_forwards_context_management_with_the_beta_inference_sends( + edit_type: str, sent_beta: str, unsent_beta: str +) -> None: + """Anthropic counts an unsigned compaction block only under the compact_20260112 edit that inference sends, + and a clearing edit changes the count, so the count carries the request's context_management and its beta.""" + context_management: Final[dict[str, JsonValue]] = {"edits": [{"type": edit_type}]} + with respx.mock: + route: Final = respx.post("https://gateway.example/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 259}) + ) + result: Final = await AnthropicTokenCounter().count_tokens( + model_to_use="claude-opus-5-5", + messages=[{"role": "assistant", "content": [{"type": "compaction", "content": "Recipe app summary."}]}], + contents=None, + deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}}, + context_management=CountContextManagement(spec=context_management, drop_invalid_edits=False), + ) + + sent: Final = route.calls.last.request + betas: Final = sent.headers["anthropic-beta"].split(",") + assert result is not None and result.total_tokens == 259 + assert TypeAdapter(dict[str, JsonValue]).validate_json(sent.content)["context_management"] == context_management + assert (sent_beta in betas, unsent_beta in betas) == (True, False) diff --git a/tests/unit/llms/vertex_ai/count_tokens/test_context.py b/tests/unit/llms/vertex_ai/count_tokens/test_context.py index b2a0b735b0f..27e276cf70f 100644 --- a/tests/unit/llms/vertex_ai/count_tokens/test_context.py +++ b/tests/unit/llms/vertex_ai/count_tokens/test_context.py @@ -666,7 +666,8 @@ async def test_anthropic_view_counts_the_turns_after_a_compaction_block_when_an_ *_agent_turns(2, first_round=3), ) - view: Final = await inference_request_view(turns, None, "anthropic", _edits(clear_before_compact)) + with patch.object(litellm, "token_counter", side_effect=RuntimeError("tokenizer unavailable")): + view: Final = await inference_request_view(turns, None, "anthropic", _edits(clear_before_compact)) assert isinstance(view, InferenceRequestView) assert _tool_outputs_kept(view) == (True, True) diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_count_tokens_compaction_beta.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_count_tokens_compaction_beta.py deleted file mode 100644 index ae02ed3d866..00000000000 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_count_tokens_compaction_beta.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -A compacted history on the Vertex AI partner count-tokens endpoint needs the compaction beta: Vertex -validates the signed compaction block exactly like Anthropic and answers 400 without the header, which -the proxy then degrades into the local tokenizer. -""" - -import pytest - -import litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler as handler_mod -from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import ( - VertexAIPartnerModelsTokenCounter, -) -from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES - -COMPACTION_BETA = ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value -COMPACTED_HISTORY = [ - { - "role": "assistant", - "content": [ - { - "type": "compaction", - "content": "The user is building a recipe app and asked for one-sentence class descriptions.", - "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40, - } - ], - }, - {"role": "user", "content": "Now do the same for Ingredient."}, -] -PLAIN_HISTORY = [{"role": "user", "content": "Now do the same for Ingredient."}] - - -@pytest.fixture -def posted_headers(monkeypatch): - captured = {} - - async def fake_ensure_access_token(self, credentials, project_id, custom_llm_provider): - return "fake-token", "fake-project" - - class FakeResponse: - status_code = 200 - - def json(self): - return {"input_tokens": 404} - - class FakeClient: - async def post(self, url, headers=None, json=None, **kwargs): - captured.update(headers or {}) - return FakeResponse() - - monkeypatch.setattr(VertexAIPartnerModelsTokenCounter, "_ensure_access_token_async", fake_ensure_access_token) - monkeypatch.setattr(handler_mod, "get_async_httpx_client", lambda **kwargs: FakeClient()) - return captured - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("messages", "beta_expected"), - [(COMPACTED_HISTORY, COMPACTION_BETA), (PLAIN_HISTORY, None)], -) -async def test_compaction_beta_is_sent_only_with_a_compacted_history(posted_headers, messages, beta_expected): - result = await VertexAIPartnerModelsTokenCounter().handle_count_tokens_request( - model="claude-sonnet-5-5", - request_data={"model": "claude-sonnet-5-5", "messages": messages}, - litellm_params={"vertex_location": "us-east5"}, - ) - - assert result["input_tokens"] == 404 - assert posted_headers["Authorization"] == "Bearer fake-token" - assert posted_headers.get("anthropic-beta") == beta_expected From 03602bcd34b60fc0b93456f6fb7b097c028590fd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 00:34:27 -0700 Subject: [PATCH 4/6] fix(anthropic): map list-form context_management to edits for count_tokens and inference betas Count requests forwarded the OpenAI list form of context_management unchanged, while inference maps it to Anthropic edits, and the inference beta helper read the list as no edits. Both now map it once. A replayed compaction block with an empty summary no longer reaches the tokenizer. --- litellm/litellm_core_utils/token_counter.py | 5 +++- litellm/llms/anthropic/common_utils.py | 18 ++++++++++-- .../anthropic/count_tokens/transformation.py | 3 +- .../litellm_core_utils/test_token_counter.py | 28 ++++++++++++++++++ .../test_anthropic_messages_transformation.py | 15 ++++++---- ...t_anthropic_count_tokens_transformation.py | 29 +++++++++++++------ 6 files changed, 78 insertions(+), 20 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 8256e3c59e2..639f39797d9 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -871,7 +871,10 @@ def _compaction_summaries(provider_specific_fields: object) -> tuple[str, ...]: return tuple( content for block in block_fields - if block is not None and block.get("type") == "compaction" and isinstance(content := block.get("content"), str) + if block is not None + and block.get("type") == "compaction" + and isinstance(content := block.get("content"), str) + and content ) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 9380e5fc9c4..1512f871a39 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -13,7 +13,7 @@ from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar from urllib.parse import quote, urlparse import httpx -from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError +from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -119,6 +119,7 @@ _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _DECODED_JSON: Final = TypeAdapter(object) +_JSON_VALUE: Final = TypeAdapter(JsonValue) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") @@ -158,8 +159,19 @@ def requires_native_compaction_beta( ) -def context_management_betas(context_management: object) -> tuple[str, ...]: - spec: Final = as_str_mapping(context_management) +def anthropic_context_management(context_management: JsonValue) -> JsonValue: + if not isinstance(context_management, list): + return context_management + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + edits: Final = AnthropicConfig.map_openai_context_management_to_anthropic( + [entry for entry in context_management if isinstance(entry, dict)] + ) + return context_management if edits is None else _JSON_VALUE.validate_python(edits) + + +def context_management_betas(context_management: JsonValue) -> tuple[str, ...]: + spec: Final = as_str_mapping(anthropic_context_management(context_management)) edits: Final = validated_object_list(spec.get("edits")) if spec is not None else None edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits or ()) if edit is not None) return tuple( diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 2ec78f0bf0f..9c7a17c75ce 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -27,6 +27,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.common_utils import ( + anthropic_context_management, carries_signed_compaction_block, context_management_betas, merge_anthropic_beta_headers, @@ -400,7 +401,7 @@ class AnthropicCountTokensConfig: for key, value in ( ("system", counted_system), ("tools", tools), - ("context_management", context_management), + ("context_management", anthropic_context_management(context_management)), ) if value is not None } diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 31c461803fe..1317f65a964 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1670,6 +1670,34 @@ def test_token_counter_counts_chat_compaction_blocks_like_their_summary_text(): assert token_counter(model=model, messages=replayed) == token_counter(model=model, messages=as_text) +class _BosEncoder: + def encode_batch_fast(self, texts: list[str]) -> list[list[int]]: + return [[0] * (len(text.split()) + 1) for text in texts] + + +def test_token_counter_charges_nothing_for_a_chat_compaction_block_with_no_summary(): + """ + A tokenizer that prepends a BOS token charges one token for the empty string, so a replayed + compaction block with an empty summary must not reach it, like an empty thinking block. + """ + model = "anthropic/claude-sonnet-5-5" + tokenizer = {"type": "huggingface_tokenizer", "tokenizer": _BosEncoder()} + follow_up = {"role": "user", "content": "Now do the same for Ingredient."} + replayed = [ + { + "role": "assistant", + "content": None, + "provider_specific_fields": {"compaction_blocks": [{"type": "compaction", "content": ""}]}, + }, + follow_up, + ] + bare = [{"role": "assistant", "content": None}, follow_up] + + assert token_counter(model=model, messages=replayed, custom_tokenizer=tokenizer) == token_counter( + model=model, messages=bare, custom_tokenizer=tokenizer + ) + + def test_count_content_list_skips_the_tokenizer_for_empty_block_text(): """ An empty thinking block, a redacted_thinking block (no thinking text at all) and a compaction diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index 693370c021d..8e77a72cdd0 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -241,22 +241,25 @@ _CLEARING_EDIT_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_202 @pytest.mark.parametrize( - ("edits", "betas"), + ("context_management", "betas"), ( - pytest.param([{"type": "compact_20260112"}], {_COMPACT_EDIT_BETA}, id="compact"), - pytest.param([{"type": "clear_tool_uses_20250919"}], {_CLEARING_EDIT_BETA}, id="clearing"), + pytest.param({"edits": [{"type": "compact_20260112"}]}, {_COMPACT_EDIT_BETA}, id="compact"), + pytest.param({"edits": [{"type": "clear_tool_uses_20250919"}]}, {_CLEARING_EDIT_BETA}, id="clearing"), pytest.param( - [{"type": "compact_20260112"}, {"type": "clear_thinking_20251015"}], + {"edits": [{"type": "compact_20260112"}, {"type": "clear_thinking_20251015"}]}, {_COMPACT_EDIT_BETA, _CLEARING_EDIT_BETA}, id="both", ), + pytest.param( + [{"type": "compaction", "compact_threshold": 150000}], {_COMPACT_EDIT_BETA}, id="openai_list_form" + ), ), ) def test_context_management_edits_add_the_beta_each_edit_needs( - edits: list[dict[str, str]], betas: set[str], local_beta_headers_config: None + context_management: JsonValue, betas: set[str], local_beta_headers_config: None ) -> None: headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta( - headers={}, optional_params={"context_management": {"edits": edits}}, messages=[] + headers={}, optional_params={"context_management": context_management}, messages=[] ) assert set(headers["anthropic-beta"].split(",")) == betas diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index ca409ef2db2..675343ced85 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -895,26 +895,37 @@ async def test_handler_sends_the_compaction_beta_with_a_compacted_history(httpx_ @pytest.mark.asyncio @pytest.mark.usefixtures("httpx_transport_clients") @pytest.mark.parametrize( - ("edit_type", "sent_beta", "unsent_beta"), + ("context_management", "forwarded", "sent_beta", "unsent_beta"), ( - ( - "compact_20260112", + pytest.param( + {"edits": [{"type": "compact_20260112"}]}, + {"edits": [{"type": "compact_20260112"}]}, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + id="compact", ), - ( - "clear_tool_uses_20250919", + pytest.param( + {"edits": [{"type": "clear_tool_uses_20250919"}]}, + {"edits": [{"type": "clear_tool_uses_20250919"}]}, ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + id="clearing", + ), + pytest.param( + [{"type": "compaction", "compact_threshold": 150000}], + {"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}]}, + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + id="openai_list_form", ), ), ) async def test_counter_forwards_context_management_with_the_beta_inference_sends( - edit_type: str, sent_beta: str, unsent_beta: str + context_management: JsonValue, forwarded: JsonValue, sent_beta: str, unsent_beta: str ) -> None: """Anthropic counts an unsigned compaction block only under the compact_20260112 edit that inference sends, - and a clearing edit changes the count, so the count carries the request's context_management and its beta.""" - context_management: Final[dict[str, JsonValue]] = {"edits": [{"type": edit_type}]} + and a clearing edit changes the count, so the count carries the request's context_management as inference + sends it, the OpenAI list form mapped to edits, and its beta.""" with respx.mock: route: Final = respx.post("https://gateway.example/v1/messages/count_tokens").mock( return_value=httpx.Response(200, json={"input_tokens": 259}) @@ -930,5 +941,5 @@ async def test_counter_forwards_context_management_with_the_beta_inference_sends sent: Final = route.calls.last.request betas: Final = sent.headers["anthropic-beta"].split(",") assert result is not None and result.total_tokens == 259 - assert TypeAdapter(dict[str, JsonValue]).validate_json(sent.content)["context_management"] == context_management + assert TypeAdapter(dict[str, JsonValue]).validate_json(sent.content)["context_management"] == forwarded assert (sent_beta in betas, unsent_beta in betas) == (True, False) From 4550fc7ac2007573460c80249febb78fae9fcea7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:22:42 -0700 Subject: [PATCH 5/6] fix(anthropic): forward count context_management only with edits, map Vertex list-form betas, and return provider context_management --- litellm/llms/anthropic/common_utils.py | 15 ++++++++-- .../anthropic/count_tokens/transformation.py | 4 +-- .../anthropic/messages/transformation.py | 25 ++--------------- .../proxy/anthropic_endpoints/endpoints.py | 18 ++++++++++-- ...t_anthropic_count_tokens_transformation.py | 28 +++++++++++++++++++ ...artner_models_anthropic_messages_config.py | 23 +++++++++++++++ .../anthropic_endpoints/test_endpoints.py | 26 +++++++++++++++-- 7 files changed, 107 insertions(+), 32 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 1369b2c9342..db3de28020c 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -218,10 +218,19 @@ def anthropic_context_management(context_management: JsonValue) -> JsonValue: return context_management if edits is None else _JSON_VALUE.validate_python(edits) +def _context_management_edits(spec: JsonValue) -> Sequence[object]: + mapping: Final = as_str_mapping(spec) + return (validated_object_list(mapping.get("edits")) if mapping is not None else None) or () + + +def context_management_with_edits(context_management: JsonValue) -> JsonValue: + spec: Final = anthropic_context_management(context_management) + return spec if _context_management_edits(spec) else None + + def context_management_betas(context_management: JsonValue) -> tuple[str, ...]: - spec: Final = as_str_mapping(anthropic_context_management(context_management)) - edits: Final = validated_object_list(spec.get("edits")) if spec is not None else None - edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits or ()) if edit is not None) + edits: Final = _context_management_edits(anthropic_context_management(context_management)) + edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits) if edit is not None) return tuple( beta for beta, used in ( diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 260e43d8400..1c8f0c206cd 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -27,9 +27,9 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.common_utils import ( - anthropic_context_management, carries_signed_compaction_block, context_management_betas, + context_management_with_edits, is_claude_model_target, merge_anthropic_beta_headers, tools_without_custom_field, @@ -413,7 +413,7 @@ class AnthropicCountTokensConfig: drop_custom=drop_params, ), ), - ("context_management", anthropic_context_management(context_management)), + ("context_management", context_management_with_edits(context_management)), ) if value is not None } diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index 1f0ee728cd1..44cfcbb797b 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -3,7 +3,7 @@ from typing import Final from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params -from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, messages_carry_output_config, @@ -171,28 +171,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert if existing_beta: beta_values.update(b.strip() for b in existing_beta.split(",")) - # Check for context management - context_management_param: Final = optional_params.get("context_management") - if context_management_param is not None: - # Check edits array for compact_20260112 type - edits: Final = context_management_param.get("edits", []) - has_compact = False - has_other = False - - for edit in edits: - edit_type = edit.get("type", "") - if edit_type == "compact_20260112": - has_compact = True - else: - has_other = True - - # Add compact header if any compact edits exist - if has_compact: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) - - # Add context management header if any other edits exist - if has_other: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value) + beta_values.update(context_management_betas(optional_params.get("context_management"))) # Check for web search tool for tool in tools: diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1d88e133cf9..45e13c59654 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -8,7 +8,7 @@ from typing import Final from fastapi import APIRouter, Depends, HTTPException, Request, Response, status from fastapi.responses import JSONResponse -from pydantic import JsonValue, ValidationError +from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -55,6 +55,7 @@ from litellm.proxy.litellm_pre_call_utils import ( should_auto_drop_params_for_agentic_cli, ) from litellm.router import Router +from litellm.types.utils import TokenCountResponse router: Final = APIRouter() @@ -355,6 +356,15 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: } +_PROVIDER_COUNT_REPLY: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) + + +def _provider_context_management(response: TokenCountResponse) -> Mapping[str, object]: + original: Final = _PROVIDER_COUNT_REPLY.validate_python(response.original_response) + context_management: Final = original.get("context_management") if original is not None else None + return {} if context_management is None else {"context_management": context_management} + + @router.post( "/v1/messages/count_tokens", tags=["[beta] Anthropic Messages Token Counting"], @@ -434,7 +444,11 @@ async def count_tokens( log_llm_api_exception(e, litellm_call_id) return _anthropic_error_json_response(proxy_exception_from_route_error(e), request) else: - return {"input_tokens": token_response.total_tokens, **count_source_fields(token_response)} + return { + "input_tokens": token_response.total_tokens, + **_provider_context_management(token_response), + **count_source_fields(token_response), + } @router.post( diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 33b7dd2a147..aa7fa2727b4 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -991,3 +991,31 @@ async def test_counter_forwards_context_management_with_the_beta_inference_sends assert result is not None and result.total_tokens == 259 assert TypeAdapter(dict[str, JsonValue]).validate_json(sent.content)["context_management"] == forwarded assert (sent_beta in betas, unsent_beta in betas) == (True, False) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@pytest.mark.parametrize("context_management", ({}, {"edits": []}, []), ids=("empty_object", "empty_edits", "empty_list")) +async def test_counter_leaves_out_context_management_without_edits(context_management: JsonValue) -> None: + """Anthropic's count endpoint rejects a context_management with no edits, which inference accepts, so the count + leaves it out and adds no context-management beta.""" + with respx.mock: + route: Final = respx.post("https://gateway.example/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 11}) + ) + result: Final = await AnthropicTokenCounter().count_tokens( + model_to_use="claude-opus-5-5", + messages=[{"role": "user", "content": "Hello"}], + contents=None, + deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}}, + context_management=CountContextManagement(spec=context_management, drop_invalid_edits=False), + ) + + sent: Final = route.calls.last.request + betas: Final = frozenset(sent.headers.get("anthropic-beta", "").split(",")) + assert result is not None and result.total_tokens == 11 + assert "context_management" not in TypeAdapter(dict[str, JsonValue]).validate_json(sent.content) + assert not betas & { + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + } diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 5916ce9f7a1..a66c59b36b2 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -507,6 +507,29 @@ def test_both_compact_and_context_management_headers_added(): ) +def test_openai_list_form_context_management_sends_the_compact_beta(): + config: Final = VertexAIPartnerModelsAnthropicMessagesConfig() + + with ( + patch.object(config, "_ensure_access_token", return_value=("token", "test-project")), + patch.object(config, "get_complete_vertex_url", return_value="https://mock-url"), + ): + updated_headers, _ = config.validate_anthropic_messages_environment( + headers={}, + model="claude-vertex-ai-opus-4-6", + messages=[], + optional_params={"context_management": [{"type": "compaction", "compact_threshold": 150000}]}, + litellm_params={ + "vertex_ai_project": "test-project", + "vertex_ai_location": "us-central1", + "vertex_credentials": "{}", + }, + api_base=None, + ) + + assert updated_headers["anthropic-beta"].split(",") == [ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value] + + def test_validate_environment_always_refreshes_token_ignoring_stale_bearer(): """Regression: stale Authorization in shared deployment extra_headers must not skip token refresh on /v1/messages — _ensure_access_token is always called.""" diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index 91e41a1f038..c6bab49af99 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -6,6 +6,7 @@ import json import logging import unittest from collections.abc import Callable, Mapping +from types import MappingProxyType from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -660,7 +661,10 @@ async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_e def _count_on_claude( - monkeypatch: pytest.MonkeyPatch, peer: Callable[[httpx.Request], httpx.Response], headers: Mapping[str, str] + monkeypatch: pytest.MonkeyPatch, + peer: Callable[[httpx.Request], httpx.Response], + headers: Mapping[str, str], + extra_body: Mapping[str, object] = MappingProxyType({}), ) -> httpx.Response: from fastapi import FastAPI @@ -700,7 +704,7 @@ def _count_on_claude( app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth() return TestClient(app).post( "/v1/messages/count_tokens", - json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}]}, + json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}], **extra_body}, headers=dict(headers), ) @@ -728,6 +732,24 @@ def test_count_tokens_logs_one_line_with_the_callers_call_id_model_and_count( ) +def test_count_tokens_returns_the_context_management_the_provider_applied(monkeypatch: pytest.MonkeyPatch) -> None: + applied: Final = {"original_input_tokens": 640} + response: Final = _count_on_claude( + monkeypatch, + lambda request: httpx.Response( + 200, json={"input_tokens": 576, "context_management": applied}, request=request + ), + {}, + {"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}, + ) + + assert response.json() == { + "input_tokens": 576, + "context_management": applied, + "litellm_count_source": "provider", + }, response.text + + def test_count_tokens_provider_rejection_logs_its_status_and_answers_the_same_error( monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] ) -> None: From fcc88b195b9e305f25ed09c71631c4f39f9ce9e5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:58:23 -0700 Subject: [PATCH 6/6] fix(anthropic): drop an unmappable context_management list under drop_params --- .../messages/native/transformation.py | 2 ++ .../test_anthropic_messages_transformation.py | 31 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index d65b0818576..ef855eff180 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -746,6 +746,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ) if transformed_context_management is not None: anthropic_messages_optional_request_params["context_management"] = transformed_context_management + elif drops_unsupported_params(litellm_params.drop_params): + anthropic_messages_optional_request_params.pop("context_management") ####### get required params for all anthropic messages requests ###### # Lazy %s: the f-string previously stringified the entire messages diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index 1314c6d52cb..8fa9d534a8c 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -350,6 +350,37 @@ def test_context_management_edits_add_the_beta_each_edit_needs( assert set(headers["anthropic-beta"].split(",")) == betas +_UNMAPPABLE_LIST_FORM: Final = [{"type": "clear_tool_uses_20250919"}] + + +@pytest.mark.parametrize("config_class", [AnthropicMessagesConfig, VertexAIPartnerModelsAnthropicMessagesConfig]) +@pytest.mark.parametrize( + ("context_management", "drop_params", "forwarded"), + ( + pytest.param(_UNMAPPABLE_LIST_FORM, True, None, id="unmappable_dropped_under_drop_params"), + pytest.param(_UNMAPPABLE_LIST_FORM, False, _UNMAPPABLE_LIST_FORM, id="unmappable_kept_for_the_provider_400"), + pytest.param( + [{"type": "compaction"}], + True, + {"edits": [{"type": "compact_20260112"}]}, + id="mappable_still_mapped_under_drop_params", + ), + ), +) +def test_context_management_the_list_form_cannot_map_is_dropped_only_under_drop_params( + config_class: type[AnthropicMessagesConfig], context_management: JsonValue, drop_params: bool, forwarded: JsonValue +) -> None: + request: Final = config_class().transform_anthropic_messages_request( + model="claude-haiku-5-5", + messages=[{"role": "user", "content": "Hi"}], + anthropic_messages_optional_request_params={"max_tokens": 50, "context_management": context_management}, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + + assert request.get("context_management") == forwarded + + @pytest.mark.parametrize("config", _classifier_configs(), ids=type) def test_routes_whose_platform_runs_the_classifier_keep_safeguards_and_the_beta( config: AnthropicMessagesConfig, local_beta_headers_config: None