From 4550fc7ac2007573460c80249febb78fae9fcea7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:22:42 -0700 Subject: [PATCH] fix(anthropic): forward count context_management only with edits, map Vertex list-form betas, and return provider context_management --- litellm/llms/anthropic/common_utils.py | 15 ++++++++-- .../anthropic/count_tokens/transformation.py | 4 +-- .../anthropic/messages/transformation.py | 25 ++--------------- .../proxy/anthropic_endpoints/endpoints.py | 18 ++++++++++-- ...t_anthropic_count_tokens_transformation.py | 28 +++++++++++++++++++ ...artner_models_anthropic_messages_config.py | 23 +++++++++++++++ .../anthropic_endpoints/test_endpoints.py | 26 +++++++++++++++-- 7 files changed, 107 insertions(+), 32 deletions(-) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 1369b2c9342..db3de28020c 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -218,10 +218,19 @@ def anthropic_context_management(context_management: JsonValue) -> JsonValue: return context_management if edits is None else _JSON_VALUE.validate_python(edits) +def _context_management_edits(spec: JsonValue) -> Sequence[object]: + mapping: Final = as_str_mapping(spec) + return (validated_object_list(mapping.get("edits")) if mapping is not None else None) or () + + +def context_management_with_edits(context_management: JsonValue) -> JsonValue: + spec: Final = anthropic_context_management(context_management) + return spec if _context_management_edits(spec) else None + + def context_management_betas(context_management: JsonValue) -> tuple[str, ...]: - spec: Final = as_str_mapping(anthropic_context_management(context_management)) - edits: Final = validated_object_list(spec.get("edits")) if spec is not None else None - edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits or ()) if edit is not None) + edits: Final = _context_management_edits(anthropic_context_management(context_management)) + edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits) if edit is not None) return tuple( beta for beta, used in ( diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 260e43d8400..1c8f0c206cd 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -27,9 +27,9 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.common_utils import ( - anthropic_context_management, carries_signed_compaction_block, context_management_betas, + context_management_with_edits, is_claude_model_target, merge_anthropic_beta_headers, tools_without_custom_field, @@ -413,7 +413,7 @@ class AnthropicCountTokensConfig: drop_custom=drop_params, ), ), - ("context_management", anthropic_context_management(context_management)), + ("context_management", context_management_with_edits(context_management)), ) if value is not None } diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index 1f0ee728cd1..44cfcbb797b 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -3,7 +3,7 @@ from typing import Final from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params -from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, messages_carry_output_config, @@ -171,28 +171,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert if existing_beta: beta_values.update(b.strip() for b in existing_beta.split(",")) - # Check for context management - context_management_param: Final = optional_params.get("context_management") - if context_management_param is not None: - # Check edits array for compact_20260112 type - edits: Final = context_management_param.get("edits", []) - has_compact = False - has_other = False - - for edit in edits: - edit_type = edit.get("type", "") - if edit_type == "compact_20260112": - has_compact = True - else: - has_other = True - - # Add compact header if any compact edits exist - if has_compact: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) - - # Add context management header if any other edits exist - if has_other: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value) + beta_values.update(context_management_betas(optional_params.get("context_management"))) # Check for web search tool for tool in tools: diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1d88e133cf9..45e13c59654 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -8,7 +8,7 @@ from typing import Final from fastapi import APIRouter, Depends, HTTPException, Request, Response, status from fastapi.responses import JSONResponse -from pydantic import JsonValue, ValidationError +from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -55,6 +55,7 @@ from litellm.proxy.litellm_pre_call_utils import ( should_auto_drop_params_for_agentic_cli, ) from litellm.router import Router +from litellm.types.utils import TokenCountResponse router: Final = APIRouter() @@ -355,6 +356,15 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: } +_PROVIDER_COUNT_REPLY: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) + + +def _provider_context_management(response: TokenCountResponse) -> Mapping[str, object]: + original: Final = _PROVIDER_COUNT_REPLY.validate_python(response.original_response) + context_management: Final = original.get("context_management") if original is not None else None + return {} if context_management is None else {"context_management": context_management} + + @router.post( "/v1/messages/count_tokens", tags=["[beta] Anthropic Messages Token Counting"], @@ -434,7 +444,11 @@ async def count_tokens( log_llm_api_exception(e, litellm_call_id) return _anthropic_error_json_response(proxy_exception_from_route_error(e), request) else: - return {"input_tokens": token_response.total_tokens, **count_source_fields(token_response)} + return { + "input_tokens": token_response.total_tokens, + **_provider_context_management(token_response), + **count_source_fields(token_response), + } @router.post( diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index 33b7dd2a147..aa7fa2727b4 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -991,3 +991,31 @@ async def test_counter_forwards_context_management_with_the_beta_inference_sends assert result is not None and result.total_tokens == 259 assert TypeAdapter(dict[str, JsonValue]).validate_json(sent.content)["context_management"] == forwarded assert (sent_beta in betas, unsent_beta in betas) == (True, False) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@pytest.mark.parametrize("context_management", ({}, {"edits": []}, []), ids=("empty_object", "empty_edits", "empty_list")) +async def test_counter_leaves_out_context_management_without_edits(context_management: JsonValue) -> None: + """Anthropic's count endpoint rejects a context_management with no edits, which inference accepts, so the count + leaves it out and adds no context-management beta.""" + with respx.mock: + route: Final = respx.post("https://gateway.example/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 11}) + ) + result: Final = await AnthropicTokenCounter().count_tokens( + model_to_use="claude-opus-5-5", + messages=[{"role": "user", "content": "Hello"}], + contents=None, + deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}}, + context_management=CountContextManagement(spec=context_management, drop_invalid_edits=False), + ) + + sent: Final = route.calls.last.request + betas: Final = frozenset(sent.headers.get("anthropic-beta", "").split(",")) + assert result is not None and result.total_tokens == 11 + assert "context_management" not in TypeAdapter(dict[str, JsonValue]).validate_json(sent.content) + assert not betas & { + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + } diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 5916ce9f7a1..a66c59b36b2 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -507,6 +507,29 @@ def test_both_compact_and_context_management_headers_added(): ) +def test_openai_list_form_context_management_sends_the_compact_beta(): + config: Final = VertexAIPartnerModelsAnthropicMessagesConfig() + + with ( + patch.object(config, "_ensure_access_token", return_value=("token", "test-project")), + patch.object(config, "get_complete_vertex_url", return_value="https://mock-url"), + ): + updated_headers, _ = config.validate_anthropic_messages_environment( + headers={}, + model="claude-vertex-ai-opus-4-6", + messages=[], + optional_params={"context_management": [{"type": "compaction", "compact_threshold": 150000}]}, + litellm_params={ + "vertex_ai_project": "test-project", + "vertex_ai_location": "us-central1", + "vertex_credentials": "{}", + }, + api_base=None, + ) + + assert updated_headers["anthropic-beta"].split(",") == [ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value] + + def test_validate_environment_always_refreshes_token_ignoring_stale_bearer(): """Regression: stale Authorization in shared deployment extra_headers must not skip token refresh on /v1/messages — _ensure_access_token is always called.""" diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index 91e41a1f038..c6bab49af99 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -6,6 +6,7 @@ import json import logging import unittest from collections.abc import Callable, Mapping +from types import MappingProxyType from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -660,7 +661,10 @@ async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_e def _count_on_claude( - monkeypatch: pytest.MonkeyPatch, peer: Callable[[httpx.Request], httpx.Response], headers: Mapping[str, str] + monkeypatch: pytest.MonkeyPatch, + peer: Callable[[httpx.Request], httpx.Response], + headers: Mapping[str, str], + extra_body: Mapping[str, object] = MappingProxyType({}), ) -> httpx.Response: from fastapi import FastAPI @@ -700,7 +704,7 @@ def _count_on_claude( app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth() return TestClient(app).post( "/v1/messages/count_tokens", - json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}]}, + json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}], **extra_body}, headers=dict(headers), ) @@ -728,6 +732,24 @@ def test_count_tokens_logs_one_line_with_the_callers_call_id_model_and_count( ) +def test_count_tokens_returns_the_context_management_the_provider_applied(monkeypatch: pytest.MonkeyPatch) -> None: + applied: Final = {"original_input_tokens": 640} + response: Final = _count_on_claude( + monkeypatch, + lambda request: httpx.Response( + 200, json={"input_tokens": 576, "context_management": applied}, request=request + ), + {}, + {"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}, + ) + + assert response.json() == { + "input_tokens": 576, + "context_management": applied, + "litellm_count_source": "provider", + }, response.text + + def test_count_tokens_provider_rejection_logs_its_status_and_answers_the_same_error( monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] ) -> None: