From 8aca5e9b3882d2fabc298423995cd9c94ffce455 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:20:10 -0700 Subject: [PATCH] fix(count_tokens): forward context edits on Vertex and Azure Claude counts and keep Bedrock's mantle answer without edits --- .../anthropic/count_tokens/handler.py | 3 ++ .../anthropic/count_tokens/token_counter.py | 1 + .../anthropic/count_tokens/transformation.py | 17 ++++++- .../count_tokens/bedrock_token_counter.py | 7 ++- .../bedrock/count_tokens/transformation.py | 2 + litellm/llms/vertex_ai/common_utils.py | 4 ++ .../count_tokens/handler.py | 15 ++++-- .../vertex_ai_partner_models/main.py | 11 +++- .../count_tokens/test_token_counter.py | 31 ++++++++++- .../test_bedrock_token_counter.py | 44 ++++++++++++++++ .../vertex_ai/test_vertex_ai_common_utils.py | 51 +++++++++++++++++++ 11 files changed, 177 insertions(+), 9 deletions(-) diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index c324f0f0980..65cbcf65a55 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -39,6 +39,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): tools: list[dict[str, JsonValue]] | None = None, system: object = None, optional_params: Mapping[str, JsonValue] | None = None, + context_management: JsonValue = None, ) -> dict[str, object]: """ Handle a CountTokens request using httpx with Azure authentication. @@ -70,6 +71,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): tools=tools, system=system, optional_params=optional_params, + context_management=context_management, drop_params=drops_unsupported_params((litellm_params or {}).get("drop_params")), ) @@ -85,6 +87,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): api_key=api_key, litellm_params=litellm_params, messages=messages, + context_management=context_management, ) if not {"x-api-key", "Authorization"} & headers.keys(): raise AnthropicError( diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py index af7cf9d81ce..14f7d4bfd2b 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py @@ -135,6 +135,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter): tools=tools, system=system, optional_params=count_options, + context_management=None if context_management is None else context_management.spec, ) if result is not None: diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py b/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py index c02c92a95d6..91ed40d9271 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py @@ -8,9 +8,16 @@ from collections.abc import Sequence from typing import Final from urllib.parse import urlsplit +from pydantic import JsonValue + +from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.anthropic.common_utils import count_tokens_feature_betas, merge_anthropic_beta_headers +from litellm.llms.anthropic.common_utils import ( + context_management_betas, + count_tokens_feature_betas, + merge_anthropic_beta_headers, +) from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountTokensConfig, ) @@ -31,6 +38,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig): api_key: str | None, litellm_params: dict[str, object] | None = None, messages: Sequence[object] = (), + context_management: JsonValue = None, ) -> dict[str, str]: """ Get the required headers for the Azure AI Anthropic CountTokens API. @@ -42,6 +50,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig): api_key: The Azure AI API key litellm_params: Optional LiteLLM parameters for additional auth config messages: The request messages, read for betas the counted features need + context_management: The edit spec the count forwards, read for the betas its edits need Returns: Dictionary of required headers with both x-api-key and Azure authentication @@ -51,7 +60,11 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig): "Content-Type": "application/json", "anthropic-version": "2023-06-01", "anthropic-beta": merge_anthropic_beta_headers( - ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, count_tokens_feature_betas("azure_ai", {}, messages) + ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, + ( + *count_tokens_feature_betas("azure_ai", {}, messages), + *filter_and_transform_beta_headers(list(context_management_betas(context_management)), "azure_ai"), + ), ), **({"x-api-key": api_key} if api_key else {}), } diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index 5f9023a2b0d..aedb4e1604d 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -79,9 +79,11 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model: ) -def _mantle_retry_outcome(runtime: CountTokensOutcome, mantle: CountTokensOutcome) -> CountTokensOutcome: +def _mantle_retry_outcome( + runtime: CountTokensOutcome, mantle: CountTokensOutcome, carried_edits: bool +) -> CountTokensOutcome: match runtime, mantle: - case CountTokensFailure(status_code=400), CountTokensFailure(status_code=404): + case CountTokensFailure(status_code=400), CountTokensFailure(status_code=404) if carried_edits: return runtime case _: return mantle @@ -228,6 +230,7 @@ class BedrockTokenCounter(BaseTokenCounter): await self._count_with( self._mantle_handler, MANTLE_TOKENIZER_TYPE, mantle_request, litellm_params, resolved_model ), + carried_edits=bool(native_context_management), ) if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model) else primary diff --git a/litellm/llms/bedrock/count_tokens/transformation.py b/litellm/llms/bedrock/count_tokens/transformation.py index df5a3827c1f..588e8a37f83 100644 --- a/litellm/llms/bedrock/count_tokens/transformation.py +++ b/litellm/llms/bedrock/count_tokens/transformation.py @@ -87,6 +87,8 @@ class BedrockCountTokensConfig(BaseAWSLLM): Returns: 'converse' or 'invokeModel' """ + if "context_management" in request_data: + return "invokeModel" messages: Final = request_data.get("messages") if isinstance(messages, list): # Anthropic content blocks carry a "type" key ({"type": "text", ...}); diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index cf090507e0d..52345e1ff40 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1458,6 +1458,7 @@ class VertexAITokenCounter(BaseTokenCounter): ) -> TokenCountResponse | None: import copy + from litellm.llms.anthropic.common_utils import context_management_with_edits from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import count_tokens_publisher from litellm.llms.vertex_ai.vertex_ai_partner_models.main import ( VertexAIError as PartnerVertexAIError, @@ -1539,6 +1540,9 @@ class VertexAITokenCounter(BaseTokenCounter): system=system, tools=tools, count_options=count_options, + context_management=( + None if context_management is None else context_management_with_edits(context_management.spec) + ), ) except (PartnerVertexAIError, httpx.HTTPStatusError) as e: status_code: Final = e.response.status_code diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index e341c8f5f48..5b010c0160a 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -9,12 +9,14 @@ their respective publisher-specific count-tokens endpoints. from collections.abc import Mapping, Sequence from typing import Any, Final -from pydantic import TypeAdapter +from pydantic import JsonValue, TypeAdapter +from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.common_utils import ( OBJECT_LIST_ADAPTER, + context_management_betas, count_tokens_feature_betas, is_claude_model_target, tools_without_custom_field, @@ -31,6 +33,7 @@ _COUNT_TOKENS_PUBLISHERS: Final = ( _REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object]) +_CONTEXT_MANAGEMENT: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) def count_tokens_request_body( @@ -55,8 +58,14 @@ def count_tokens_publisher(model: str) -> str | None: def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]: authorization: Final = {"Authorization": f"Bearer {access_token}"} - betas: Final = count_tokens_feature_betas( - "vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"]) + betas: Final = ( + *count_tokens_feature_betas( + "vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"]) + ), + *filter_and_transform_beta_headers( + list(context_management_betas(_CONTEXT_MANAGEMENT.validate_python(request_data.get("context_management")))), + "vertex_ai", + ), ) return {**authorization, "anthropic-beta": ",".join(betas)} if betas else authorization diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py index a3a3427670d..c298ca404e2 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py @@ -268,6 +268,7 @@ class VertexAIPartnerModels(VertexBase): system: object | None = None, tools: list[dict[str, object]] | None = None, count_options: Mapping[str, object] = MappingProxyType({}), + context_management: object = None, ): """ Count tokens for Vertex AI partner models (Anthropic Claude, Mistral, etc.) @@ -302,7 +303,15 @@ class VertexAIPartnerModels(VertexBase): "model": model, "messages": messages, **MappingProxyType( - {key: value for key, value in (("system", system), ("tools", tools)) if value is not None} + { + key: value + for key, value in ( + ("system", system), + ("tools", tools), + ("context_management", context_management), + ) + if value is not None + } ), **count_options, } diff --git a/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py b/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py index 7807975be24..6ca4922fa6e 100644 --- a/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py +++ b/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py @@ -9,7 +9,7 @@ from pydantic import ValidationError import litellm from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAnthropicTokenCounter -from litellm.types.utils import TokenCountResponse +from litellm.types.utils import CountContextManagement, TokenCountResponse API_BASE: Final = "https://my-resource.services.ai.azure.com" COUNT_URL: Final = f"{API_BASE}/anthropic/v1/messages/count_tokens" @@ -268,3 +268,32 @@ async def test_count_sends_the_tool_change_beta_when_the_history_adds_a_tool(mon assert result is not None assert result.total_tokens == 541 assert route.calls.last.request.headers["anthropic-beta"] == "mid-conversation-tool-changes-2026-07-01,token-counting-2024-11-01" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("edit_type", "expected_beta"), + [ + ("clear_tool_uses_20250919", "context-management-2025-06-27,token-counting-2024-11-01"), + ("compact_20260112", "token-counting-2024-11-01"), + ], +) +async def test_count_forwards_the_context_edits_with_the_betas_azure_inference_sends( + monkeypatch: pytest.MonkeyPatch, edit_type: str, expected_beta: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + spec: Final = {"edits": [{"type": edit_type}]} + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 88})) + result: Final = await AzureAIAnthropicTokenCounter().count_tokens( + model_to_use="claude-sonnet-5-5", + messages=[{"role": "user", "content": "Summarize the thread."}], + contents=None, + deployment={"litellm_params": {"api_key": "azure-key", "api_base": API_BASE}}, + request_model="azure_ai/claude-sonnet-5-5", + context_management=CountContextManagement(spec=spec, drop_invalid_edits=False), + ) + assert result is not None + assert result.total_tokens == 88 + assert json.loads(route.calls.last.request.content)["context_management"] == spec + assert route.calls.last.request.headers["anthropic-beta"] == expected_beta diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py index 946d4d6d529..739f7240916 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_token_counter.py @@ -172,6 +172,50 @@ async def test_runtime_rejection_of_an_edit_stands_when_mantle_does_not_serve_th assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST) +@pytest.mark.asyncio +async def test_runtime_400_without_an_edit_still_reports_the_mantle_failure() -> None: + bedrock: Final = _Bedrock( + runtime=(400, {"message": "The provided model identifier is invalid."}), + mantle=(404, {"type": "error", "error": {"type": "not_found_error", "message": "does not exist"}}), + ) + + result: Final = await _counter(bedrock).count_tokens( + model_to_use="us.anthropic.claude-haiku-4-5-20251001-v1:0", + messages=MESSAGES, + contents=None, + deployment=DEPLOYMENT, + request_model="claude-haiku-4-5", + ) + + assert result is not None + assert result.status_code == 404 + assert result.tokenizer_type == "bedrock_mantle_api" + assert "does not exist" in (result.error_message or "") + assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST) + + +@pytest.mark.asyncio +async def test_plain_text_history_with_an_edit_is_counted_through_invoke_carrying_the_edit() -> None: + bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 57}), mantle=(200, {"input_tokens": 1})) + spec: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]} + + result: Final = await _counter(bedrock).count_tokens( + model_to_use="global.anthropic.claude-opus-4-8", + messages=MESSAGES, + contents=None, + deployment=DEPLOYMENT, + request_model="claude-opus-4-8", + context_management=CountContextManagement(spec=spec, drop_invalid_edits=False), + ) + + assert result is not None + assert result.total_tokens == 57 + assert bedrock.posted_hosts() == (RUNTIME_HOST,) + invoke_body: Final = _invoke_body(bedrock.requests[0]) + assert invoke_body["context_management"] == spec + assert invoke_body["anthropic_beta"] == ["context-management-2025-06-27"] + + @pytest.mark.asyncio async def test_claude_runtime_404_retries_with_system_and_tools_on_mantle() -> None: bedrock: Final = _Bedrock(runtime=(404, UNSUPPORTED), mantle=(200, {"input_tokens": 2177})) diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py index 3b222be6e5e..cfbd69231d0 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -1826,6 +1826,57 @@ async def test_vertex_ai_token_counter_forwards_system_and_tools_to_partner_requ assert without_optional_fields.total_tokens == 37 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("edit_type", "expected_beta"), + [("clear_tool_uses_20250919", "context-management-2025-06-27"), ("compact_20260112", "compact-2026-01-12")], +) +async def test_vertex_claude_count_forwards_the_context_edits_and_beta_vertex_inference_sends( + edit_type: str, expected_beta: str +) -> None: + from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler + + class FakeResponse: + status_code = 200 + + def json(self) -> dict[str, int]: + return {"input_tokens": 21} + + class FakeHttpClient: + posted: tuple[tuple[dict[str, str], dict[str, object]], ...] = () + + async def post( + self, url: str, headers: dict[str, str], json: dict[str, object], timeout: float + ) -> FakeResponse: + self.posted = (*self.posted, (headers, json)) + return FakeResponse() + + fake_http_client: Final = FakeHttpClient() + spec: Final[dict[str, JsonValue]] = {"edits": [{"type": edit_type}]} + with ( + patch.object(handler, "get_async_httpx_client", return_value=fake_http_client), + patch.object( + handler.VertexAIPartnerModelsTokenCounter, + "_ensure_access_token_async", + new=AsyncMock(return_value=("fake-token", "test-project")), + ), + ): + result: Final = await VertexAITokenCounter().count_tokens( + model_to_use="claude-opus-4-8", + messages=[{"role": "user", "content": "Hello"}], + contents=None, + deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}}, + messages_format="anthropic", + context_management=CountContextManagement(spec=spec, drop_invalid_edits=False), + ) + + assert result is not None + assert result.total_tokens == 21 + ((headers, body),) = fake_http_client.posted + assert body["context_management"] == spec + assert headers["anthropic-beta"] == expected_beta + + _PARTNER_CHAT_TOOL_TURNS: Final[list[dict[str, object]]] = [ {"role": "user", "content": "Weather in SF?"}, {