fix(count_tokens): forward context edits on Vertex and Azure Claude counts and keep Bedrock's mantle answer without edits

This commit is contained in:
mateo-berri 2026-10-31 00:20:10 -07:00
parent 62e8e7ac3f
commit 8aca5e9b38
11 changed files with 177 additions and 9 deletions

View file

@ -39,6 +39,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
tools: list[dict[str, JsonValue]] | None = None,
system: object = None,
optional_params: Mapping[str, JsonValue] | None = None,
context_management: JsonValue = None,
) -> dict[str, object]:
"""
Handle a CountTokens request using httpx with Azure authentication.
@ -70,6 +71,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
tools=tools,
system=system,
optional_params=optional_params,
context_management=context_management,
drop_params=drops_unsupported_params((litellm_params or {}).get("drop_params")),
)
@ -85,6 +87,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
api_key=api_key,
litellm_params=litellm_params,
messages=messages,
context_management=context_management,
)
if not {"x-api-key", "Authorization"} & headers.keys():
raise AnthropicError(

View file

@ -135,6 +135,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
tools=tools,
system=system,
optional_params=count_options,
context_management=None if context_management is None else context_management.spec,
)
if result is not None:

View file

@ -8,9 +8,16 @@ from collections.abc import Sequence
from typing import Final
from urllib.parse import urlsplit
from pydantic import JsonValue
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import count_tokens_feature_betas, merge_anthropic_beta_headers
from litellm.llms.anthropic.common_utils import (
context_management_betas,
count_tokens_feature_betas,
merge_anthropic_beta_headers,
)
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountTokensConfig,
)
@ -31,6 +38,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
api_key: str | None,
litellm_params: dict[str, object] | None = None,
messages: Sequence[object] = (),
context_management: JsonValue = None,
) -> dict[str, str]:
"""
Get the required headers for the Azure AI Anthropic CountTokens API.
@ -42,6 +50,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
api_key: The Azure AI API key
litellm_params: Optional LiteLLM parameters for additional auth config
messages: The request messages, read for betas the counted features need
context_management: The edit spec the count forwards, read for the betas its edits need
Returns:
Dictionary of required headers with both x-api-key and Azure authentication
@ -51,7 +60,11 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
"Content-Type": "application/json",
"anthropic-version": "2023-06-01",
"anthropic-beta": merge_anthropic_beta_headers(
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, count_tokens_feature_betas("azure_ai", {}, messages)
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
(
*count_tokens_feature_betas("azure_ai", {}, messages),
*filter_and_transform_beta_headers(list(context_management_betas(context_management)), "azure_ai"),
),
),
**({"x-api-key": api_key} if api_key else {}),
}

View file

@ -79,9 +79,11 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model:
)
def _mantle_retry_outcome(runtime: CountTokensOutcome, mantle: CountTokensOutcome) -> CountTokensOutcome:
def _mantle_retry_outcome(
runtime: CountTokensOutcome, mantle: CountTokensOutcome, carried_edits: bool
) -> CountTokensOutcome:
match runtime, mantle:
case CountTokensFailure(status_code=400), CountTokensFailure(status_code=404):
case CountTokensFailure(status_code=400), CountTokensFailure(status_code=404) if carried_edits:
return runtime
case _:
return mantle
@ -228,6 +230,7 @@ class BedrockTokenCounter(BaseTokenCounter):
await self._count_with(
self._mantle_handler, MANTLE_TOKENIZER_TYPE, mantle_request, litellm_params, resolved_model
),
carried_edits=bool(native_context_management),
)
if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model)
else primary

View file

@ -87,6 +87,8 @@ class BedrockCountTokensConfig(BaseAWSLLM):
Returns:
'converse' or 'invokeModel'
"""
if "context_management" in request_data:
return "invokeModel"
messages: Final = request_data.get("messages")
if isinstance(messages, list):
# Anthropic content blocks carry a "type" key ({"type": "text", ...});

View file

@ -1458,6 +1458,7 @@ class VertexAITokenCounter(BaseTokenCounter):
) -> TokenCountResponse | None:
import copy
from litellm.llms.anthropic.common_utils import context_management_with_edits
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import count_tokens_publisher
from litellm.llms.vertex_ai.vertex_ai_partner_models.main import (
VertexAIError as PartnerVertexAIError,
@ -1539,6 +1540,9 @@ class VertexAITokenCounter(BaseTokenCounter):
system=system,
tools=tools,
count_options=count_options,
context_management=(
None if context_management is None else context_management_with_edits(context_management.spec)
),
)
except (PartnerVertexAIError, httpx.HTTPStatusError) as e:
status_code: Final = e.response.status_code

View file

@ -9,12 +9,14 @@ their respective publisher-specific count-tokens endpoints.
from collections.abc import Mapping, Sequence
from typing import Any, Final
from pydantic import TypeAdapter
from pydantic import JsonValue, TypeAdapter
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import (
OBJECT_LIST_ADAPTER,
context_management_betas,
count_tokens_feature_betas,
is_claude_model_target,
tools_without_custom_field,
@ -31,6 +33,7 @@ _COUNT_TOKENS_PUBLISHERS: Final = (
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
_CONTEXT_MANAGEMENT: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
def count_tokens_request_body(
@ -55,8 +58,14 @@ def count_tokens_publisher(model: str) -> str | None:
def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]:
authorization: Final = {"Authorization": f"Bearer {access_token}"}
betas: Final = count_tokens_feature_betas(
"vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
betas: Final = (
*count_tokens_feature_betas(
"vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
),
*filter_and_transform_beta_headers(
list(context_management_betas(_CONTEXT_MANAGEMENT.validate_python(request_data.get("context_management")))),
"vertex_ai",
),
)
return {**authorization, "anthropic-beta": ",".join(betas)} if betas else authorization

View file

@ -268,6 +268,7 @@ class VertexAIPartnerModels(VertexBase):
system: object | None = None,
tools: list[dict[str, object]] | None = None,
count_options: Mapping[str, object] = MappingProxyType({}),
context_management: object = None,
):
"""
Count tokens for Vertex AI partner models (Anthropic Claude, Mistral, etc.)
@ -302,7 +303,15 @@ class VertexAIPartnerModels(VertexBase):
"model": model,
"messages": messages,
**MappingProxyType(
{key: value for key, value in (("system", system), ("tools", tools)) if value is not None}
{
key: value
for key, value in (
("system", system),
("tools", tools),
("context_management", context_management),
)
if value is not None
}
),
**count_options,
}

View file

@ -9,7 +9,7 @@ from pydantic import ValidationError
import litellm
from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAnthropicTokenCounter
from litellm.types.utils import TokenCountResponse
from litellm.types.utils import CountContextManagement, TokenCountResponse
API_BASE: Final = "https://my-resource.services.ai.azure.com"
COUNT_URL: Final = f"{API_BASE}/anthropic/v1/messages/count_tokens"
@ -268,3 +268,32 @@ async def test_count_sends_the_tool_change_beta_when_the_history_adds_a_tool(mon
assert result is not None
assert result.total_tokens == 541
assert route.calls.last.request.headers["anthropic-beta"] == "mid-conversation-tool-changes-2026-07-01,token-counting-2024-11-01"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("edit_type", "expected_beta"),
[
("clear_tool_uses_20250919", "context-management-2025-06-27,token-counting-2024-11-01"),
("compact_20260112", "token-counting-2024-11-01"),
],
)
async def test_count_forwards_the_context_edits_with_the_betas_azure_inference_sends(
monkeypatch: pytest.MonkeyPatch, edit_type: str, expected_beta: str
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
spec: Final = {"edits": [{"type": edit_type}]}
with respx.mock:
route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 88}))
result: Final = await AzureAIAnthropicTokenCounter().count_tokens(
model_to_use="claude-sonnet-5-5",
messages=[{"role": "user", "content": "Summarize the thread."}],
contents=None,
deployment={"litellm_params": {"api_key": "azure-key", "api_base": API_BASE}},
request_model="azure_ai/claude-sonnet-5-5",
context_management=CountContextManagement(spec=spec, drop_invalid_edits=False),
)
assert result is not None
assert result.total_tokens == 88
assert json.loads(route.calls.last.request.content)["context_management"] == spec
assert route.calls.last.request.headers["anthropic-beta"] == expected_beta

View file

@ -172,6 +172,50 @@ async def test_runtime_rejection_of_an_edit_stands_when_mantle_does_not_serve_th
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
@pytest.mark.asyncio
async def test_runtime_400_without_an_edit_still_reports_the_mantle_failure() -> None:
bedrock: Final = _Bedrock(
runtime=(400, {"message": "The provided model identifier is invalid."}),
mantle=(404, {"type": "error", "error": {"type": "not_found_error", "message": "does not exist"}}),
)
result: Final = await _counter(bedrock).count_tokens(
model_to_use="us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=MESSAGES,
contents=None,
deployment=DEPLOYMENT,
request_model="claude-haiku-4-5",
)
assert result is not None
assert result.status_code == 404
assert result.tokenizer_type == "bedrock_mantle_api"
assert "does not exist" in (result.error_message or "")
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
@pytest.mark.asyncio
async def test_plain_text_history_with_an_edit_is_counted_through_invoke_carrying_the_edit() -> None:
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 57}), mantle=(200, {"input_tokens": 1}))
spec: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]}
result: Final = await _counter(bedrock).count_tokens(
model_to_use="global.anthropic.claude-opus-4-8",
messages=MESSAGES,
contents=None,
deployment=DEPLOYMENT,
request_model="claude-opus-4-8",
context_management=CountContextManagement(spec=spec, drop_invalid_edits=False),
)
assert result is not None
assert result.total_tokens == 57
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
invoke_body: Final = _invoke_body(bedrock.requests[0])
assert invoke_body["context_management"] == spec
assert invoke_body["anthropic_beta"] == ["context-management-2025-06-27"]
@pytest.mark.asyncio
async def test_claude_runtime_404_retries_with_system_and_tools_on_mantle() -> None:
bedrock: Final = _Bedrock(runtime=(404, UNSUPPORTED), mantle=(200, {"input_tokens": 2177}))

View file

@ -1826,6 +1826,57 @@ async def test_vertex_ai_token_counter_forwards_system_and_tools_to_partner_requ
assert without_optional_fields.total_tokens == 37
@pytest.mark.asyncio
@pytest.mark.parametrize(
("edit_type", "expected_beta"),
[("clear_tool_uses_20250919", "context-management-2025-06-27"), ("compact_20260112", "compact-2026-01-12")],
)
async def test_vertex_claude_count_forwards_the_context_edits_and_beta_vertex_inference_sends(
edit_type: str, expected_beta: str
) -> None:
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler
class FakeResponse:
status_code = 200
def json(self) -> dict[str, int]:
return {"input_tokens": 21}
class FakeHttpClient:
posted: tuple[tuple[dict[str, str], dict[str, object]], ...] = ()
async def post(
self, url: str, headers: dict[str, str], json: dict[str, object], timeout: float
) -> FakeResponse:
self.posted = (*self.posted, (headers, json))
return FakeResponse()
fake_http_client: Final = FakeHttpClient()
spec: Final[dict[str, JsonValue]] = {"edits": [{"type": edit_type}]}
with (
patch.object(handler, "get_async_httpx_client", return_value=fake_http_client),
patch.object(
handler.VertexAIPartnerModelsTokenCounter,
"_ensure_access_token_async",
new=AsyncMock(return_value=("fake-token", "test-project")),
),
):
result: Final = await VertexAITokenCounter().count_tokens(
model_to_use="claude-opus-4-8",
messages=[{"role": "user", "content": "Hello"}],
contents=None,
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
messages_format="anthropic",
context_management=CountContextManagement(spec=spec, drop_invalid_edits=False),
)
assert result is not None
assert result.total_tokens == 21
((headers, body),) = fake_http_client.posted
assert body["context_management"] == spec
assert headers["anthropic-beta"] == expected_beta
_PARTNER_CHAT_TOOL_TURNS: Final[list[dict[str, object]]] = [
{"role": "user", "content": "Weather in SF?"},
{