mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(count_tokens): forward context edits on Vertex and Azure Claude counts and keep Bedrock's mantle answer without edits
This commit is contained in:
parent
62e8e7ac3f
commit
8aca5e9b38
11 changed files with 177 additions and 9 deletions
|
|
@ -39,6 +39,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
|
|||
tools: list[dict[str, JsonValue]] | None = None,
|
||||
system: object = None,
|
||||
optional_params: Mapping[str, JsonValue] | None = None,
|
||||
context_management: JsonValue = None,
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Handle a CountTokens request using httpx with Azure authentication.
|
||||
|
|
@ -70,6 +71,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
|
|||
tools=tools,
|
||||
system=system,
|
||||
optional_params=optional_params,
|
||||
context_management=context_management,
|
||||
drop_params=drops_unsupported_params((litellm_params or {}).get("drop_params")),
|
||||
)
|
||||
|
||||
|
|
@ -85,6 +87,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
|
|||
api_key=api_key,
|
||||
litellm_params=litellm_params,
|
||||
messages=messages,
|
||||
context_management=context_management,
|
||||
)
|
||||
if not {"x-api-key", "Authorization"} & headers.keys():
|
||||
raise AnthropicError(
|
||||
|
|
|
|||
|
|
@ -135,6 +135,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
|
|||
tools=tools,
|
||||
system=system,
|
||||
optional_params=count_options,
|
||||
context_management=None if context_management is None else context_management.spec,
|
||||
)
|
||||
|
||||
if result is not None:
|
||||
|
|
|
|||
|
|
@ -8,9 +8,16 @@ from collections.abc import Sequence
|
|||
from typing import Final
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
|
||||
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.common_utils import count_tokens_feature_betas, merge_anthropic_beta_headers
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
context_management_betas,
|
||||
count_tokens_feature_betas,
|
||||
merge_anthropic_beta_headers,
|
||||
)
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
AnthropicCountTokensConfig,
|
||||
)
|
||||
|
|
@ -31,6 +38,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
|
|||
api_key: str | None,
|
||||
litellm_params: dict[str, object] | None = None,
|
||||
messages: Sequence[object] = (),
|
||||
context_management: JsonValue = None,
|
||||
) -> dict[str, str]:
|
||||
"""
|
||||
Get the required headers for the Azure AI Anthropic CountTokens API.
|
||||
|
|
@ -42,6 +50,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
|
|||
api_key: The Azure AI API key
|
||||
litellm_params: Optional LiteLLM parameters for additional auth config
|
||||
messages: The request messages, read for betas the counted features need
|
||||
context_management: The edit spec the count forwards, read for the betas its edits need
|
||||
|
||||
Returns:
|
||||
Dictionary of required headers with both x-api-key and Azure authentication
|
||||
|
|
@ -51,7 +60,11 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
|
|||
"Content-Type": "application/json",
|
||||
"anthropic-version": "2023-06-01",
|
||||
"anthropic-beta": merge_anthropic_beta_headers(
|
||||
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, count_tokens_feature_betas("azure_ai", {}, messages)
|
||||
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
|
||||
(
|
||||
*count_tokens_feature_betas("azure_ai", {}, messages),
|
||||
*filter_and_transform_beta_headers(list(context_management_betas(context_management)), "azure_ai"),
|
||||
),
|
||||
),
|
||||
**({"x-api-key": api_key} if api_key else {}),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -79,9 +79,11 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model:
|
|||
)
|
||||
|
||||
|
||||
def _mantle_retry_outcome(runtime: CountTokensOutcome, mantle: CountTokensOutcome) -> CountTokensOutcome:
|
||||
def _mantle_retry_outcome(
|
||||
runtime: CountTokensOutcome, mantle: CountTokensOutcome, carried_edits: bool
|
||||
) -> CountTokensOutcome:
|
||||
match runtime, mantle:
|
||||
case CountTokensFailure(status_code=400), CountTokensFailure(status_code=404):
|
||||
case CountTokensFailure(status_code=400), CountTokensFailure(status_code=404) if carried_edits:
|
||||
return runtime
|
||||
case _:
|
||||
return mantle
|
||||
|
|
@ -228,6 +230,7 @@ class BedrockTokenCounter(BaseTokenCounter):
|
|||
await self._count_with(
|
||||
self._mantle_handler, MANTLE_TOKENIZER_TYPE, mantle_request, litellm_params, resolved_model
|
||||
),
|
||||
carried_edits=bool(native_context_management),
|
||||
)
|
||||
if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model)
|
||||
else primary
|
||||
|
|
|
|||
|
|
@ -87,6 +87,8 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
Returns:
|
||||
'converse' or 'invokeModel'
|
||||
"""
|
||||
if "context_management" in request_data:
|
||||
return "invokeModel"
|
||||
messages: Final = request_data.get("messages")
|
||||
if isinstance(messages, list):
|
||||
# Anthropic content blocks carry a "type" key ({"type": "text", ...});
|
||||
|
|
|
|||
|
|
@ -1458,6 +1458,7 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
) -> TokenCountResponse | None:
|
||||
import copy
|
||||
|
||||
from litellm.llms.anthropic.common_utils import context_management_with_edits
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import count_tokens_publisher
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.main import (
|
||||
VertexAIError as PartnerVertexAIError,
|
||||
|
|
@ -1539,6 +1540,9 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
system=system,
|
||||
tools=tools,
|
||||
count_options=count_options,
|
||||
context_management=(
|
||||
None if context_management is None else context_management_with_edits(context_management.spec)
|
||||
),
|
||||
)
|
||||
except (PartnerVertexAIError, httpx.HTTPStatusError) as e:
|
||||
status_code: Final = e.response.status_code
|
||||
|
|
|
|||
|
|
@ -9,12 +9,14 @@ their respective publisher-specific count-tokens endpoints.
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
OBJECT_LIST_ADAPTER,
|
||||
context_management_betas,
|
||||
count_tokens_feature_betas,
|
||||
is_claude_model_target,
|
||||
tools_without_custom_field,
|
||||
|
|
@ -31,6 +33,7 @@ _COUNT_TOKENS_PUBLISHERS: Final = (
|
|||
|
||||
|
||||
_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
|
||||
_CONTEXT_MANAGEMENT: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
|
||||
|
||||
def count_tokens_request_body(
|
||||
|
|
@ -55,8 +58,14 @@ def count_tokens_publisher(model: str) -> str | None:
|
|||
|
||||
def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]:
|
||||
authorization: Final = {"Authorization": f"Bearer {access_token}"}
|
||||
betas: Final = count_tokens_feature_betas(
|
||||
"vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
|
||||
betas: Final = (
|
||||
*count_tokens_feature_betas(
|
||||
"vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
|
||||
),
|
||||
*filter_and_transform_beta_headers(
|
||||
list(context_management_betas(_CONTEXT_MANAGEMENT.validate_python(request_data.get("context_management")))),
|
||||
"vertex_ai",
|
||||
),
|
||||
)
|
||||
return {**authorization, "anthropic-beta": ",".join(betas)} if betas else authorization
|
||||
|
||||
|
|
|
|||
|
|
@ -268,6 +268,7 @@ class VertexAIPartnerModels(VertexBase):
|
|||
system: object | None = None,
|
||||
tools: list[dict[str, object]] | None = None,
|
||||
count_options: Mapping[str, object] = MappingProxyType({}),
|
||||
context_management: object = None,
|
||||
):
|
||||
"""
|
||||
Count tokens for Vertex AI partner models (Anthropic Claude, Mistral, etc.)
|
||||
|
|
@ -302,7 +303,15 @@ class VertexAIPartnerModels(VertexBase):
|
|||
"model": model,
|
||||
"messages": messages,
|
||||
**MappingProxyType(
|
||||
{key: value for key, value in (("system", system), ("tools", tools)) if value is not None}
|
||||
{
|
||||
key: value
|
||||
for key, value in (
|
||||
("system", system),
|
||||
("tools", tools),
|
||||
("context_management", context_management),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
),
|
||||
**count_options,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from pydantic import ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAnthropicTokenCounter
|
||||
from litellm.types.utils import TokenCountResponse
|
||||
from litellm.types.utils import CountContextManagement, TokenCountResponse
|
||||
|
||||
API_BASE: Final = "https://my-resource.services.ai.azure.com"
|
||||
COUNT_URL: Final = f"{API_BASE}/anthropic/v1/messages/count_tokens"
|
||||
|
|
@ -268,3 +268,32 @@ async def test_count_sends_the_tool_change_beta_when_the_history_adds_a_tool(mon
|
|||
assert result is not None
|
||||
assert result.total_tokens == 541
|
||||
assert route.calls.last.request.headers["anthropic-beta"] == "mid-conversation-tool-changes-2026-07-01,token-counting-2024-11-01"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("edit_type", "expected_beta"),
|
||||
[
|
||||
("clear_tool_uses_20250919", "context-management-2025-06-27,token-counting-2024-11-01"),
|
||||
("compact_20260112", "token-counting-2024-11-01"),
|
||||
],
|
||||
)
|
||||
async def test_count_forwards_the_context_edits_with_the_betas_azure_inference_sends(
|
||||
monkeypatch: pytest.MonkeyPatch, edit_type: str, expected_beta: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
spec: Final = {"edits": [{"type": edit_type}]}
|
||||
with respx.mock:
|
||||
route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 88}))
|
||||
result: Final = await AzureAIAnthropicTokenCounter().count_tokens(
|
||||
model_to_use="claude-sonnet-5-5",
|
||||
messages=[{"role": "user", "content": "Summarize the thread."}],
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"api_key": "azure-key", "api_base": API_BASE}},
|
||||
request_model="azure_ai/claude-sonnet-5-5",
|
||||
context_management=CountContextManagement(spec=spec, drop_invalid_edits=False),
|
||||
)
|
||||
assert result is not None
|
||||
assert result.total_tokens == 88
|
||||
assert json.loads(route.calls.last.request.content)["context_management"] == spec
|
||||
assert route.calls.last.request.headers["anthropic-beta"] == expected_beta
|
||||
|
|
|
|||
|
|
@ -172,6 +172,50 @@ async def test_runtime_rejection_of_an_edit_stands_when_mantle_does_not_serve_th
|
|||
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_400_without_an_edit_still_reports_the_mantle_failure() -> None:
|
||||
bedrock: Final = _Bedrock(
|
||||
runtime=(400, {"message": "The provided model identifier is invalid."}),
|
||||
mantle=(404, {"type": "error", "error": {"type": "not_found_error", "message": "does not exist"}}),
|
||||
)
|
||||
|
||||
result: Final = await _counter(bedrock).count_tokens(
|
||||
model_to_use="us.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
messages=MESSAGES,
|
||||
contents=None,
|
||||
deployment=DEPLOYMENT,
|
||||
request_model="claude-haiku-4-5",
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.status_code == 404
|
||||
assert result.tokenizer_type == "bedrock_mantle_api"
|
||||
assert "does not exist" in (result.error_message or "")
|
||||
assert bedrock.posted_hosts() == (RUNTIME_HOST, MANTLE_HOST)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plain_text_history_with_an_edit_is_counted_through_invoke_carrying_the_edit() -> None:
|
||||
bedrock: Final = _Bedrock(runtime=(200, {"inputTokens": 57}), mantle=(200, {"input_tokens": 1}))
|
||||
spec: Final[dict[str, JsonValue]] = {"edits": [{"type": "clear_tool_uses_20250919"}]}
|
||||
|
||||
result: Final = await _counter(bedrock).count_tokens(
|
||||
model_to_use="global.anthropic.claude-opus-4-8",
|
||||
messages=MESSAGES,
|
||||
contents=None,
|
||||
deployment=DEPLOYMENT,
|
||||
request_model="claude-opus-4-8",
|
||||
context_management=CountContextManagement(spec=spec, drop_invalid_edits=False),
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.total_tokens == 57
|
||||
assert bedrock.posted_hosts() == (RUNTIME_HOST,)
|
||||
invoke_body: Final = _invoke_body(bedrock.requests[0])
|
||||
assert invoke_body["context_management"] == spec
|
||||
assert invoke_body["anthropic_beta"] == ["context-management-2025-06-27"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claude_runtime_404_retries_with_system_and_tools_on_mantle() -> None:
|
||||
bedrock: Final = _Bedrock(runtime=(404, UNSUPPORTED), mantle=(200, {"input_tokens": 2177}))
|
||||
|
|
|
|||
|
|
@ -1826,6 +1826,57 @@ async def test_vertex_ai_token_counter_forwards_system_and_tools_to_partner_requ
|
|||
assert without_optional_fields.total_tokens == 37
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("edit_type", "expected_beta"),
|
||||
[("clear_tool_uses_20250919", "context-management-2025-06-27"), ("compact_20260112", "compact-2026-01-12")],
|
||||
)
|
||||
async def test_vertex_claude_count_forwards_the_context_edits_and_beta_vertex_inference_sends(
|
||||
edit_type: str, expected_beta: str
|
||||
) -> None:
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens import handler
|
||||
|
||||
class FakeResponse:
|
||||
status_code = 200
|
||||
|
||||
def json(self) -> dict[str, int]:
|
||||
return {"input_tokens": 21}
|
||||
|
||||
class FakeHttpClient:
|
||||
posted: tuple[tuple[dict[str, str], dict[str, object]], ...] = ()
|
||||
|
||||
async def post(
|
||||
self, url: str, headers: dict[str, str], json: dict[str, object], timeout: float
|
||||
) -> FakeResponse:
|
||||
self.posted = (*self.posted, (headers, json))
|
||||
return FakeResponse()
|
||||
|
||||
fake_http_client: Final = FakeHttpClient()
|
||||
spec: Final[dict[str, JsonValue]] = {"edits": [{"type": edit_type}]}
|
||||
with (
|
||||
patch.object(handler, "get_async_httpx_client", return_value=fake_http_client),
|
||||
patch.object(
|
||||
handler.VertexAIPartnerModelsTokenCounter,
|
||||
"_ensure_access_token_async",
|
||||
new=AsyncMock(return_value=("fake-token", "test-project")),
|
||||
),
|
||||
):
|
||||
result: Final = await VertexAITokenCounter().count_tokens(
|
||||
model_to_use="claude-opus-4-8",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"vertex_project": "test-project", "vertex_location": "us-east5"}},
|
||||
messages_format="anthropic",
|
||||
context_management=CountContextManagement(spec=spec, drop_invalid_edits=False),
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.total_tokens == 21
|
||||
((headers, body),) = fake_http_client.posted
|
||||
assert body["context_management"] == spec
|
||||
assert headers["anthropic-beta"] == expected_beta
|
||||
|
||||
|
||||
_PARTNER_CHAT_TOOL_TURNS: Final[list[dict[str, object]]] = [
|
||||
{"role": "user", "content": "Weather in SF?"},
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue