fix(anthropic): forward count context_management only with edits, map Vertex list-form betas, and return provider context_management

This commit is contained in:
mateo-berri 2026-10-29 08:22:42 -07:00
parent a3fc1485d1
commit 4550fc7ac2
7 changed files with 107 additions and 32 deletions

View file

@ -218,10 +218,19 @@ def anthropic_context_management(context_management: JsonValue) -> JsonValue:
return context_management if edits is None else _JSON_VALUE.validate_python(edits)
def _context_management_edits(spec: JsonValue) -> Sequence[object]:
mapping: Final = as_str_mapping(spec)
return (validated_object_list(mapping.get("edits")) if mapping is not None else None) or ()
def context_management_with_edits(context_management: JsonValue) -> JsonValue:
spec: Final = anthropic_context_management(context_management)
return spec if _context_management_edits(spec) else None
def context_management_betas(context_management: JsonValue) -> tuple[str, ...]:
spec: Final = as_str_mapping(anthropic_context_management(context_management))
edits: Final = validated_object_list(spec.get("edits")) if spec is not None else None
edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits or ()) if edit is not None)
edits: Final = _context_management_edits(anthropic_context_management(context_management))
edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits) if edit is not None)
return tuple(
beta
for beta, used in (

View file

@ -27,9 +27,9 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import (
anthropic_context_management,
carries_signed_compaction_block,
context_management_betas,
context_management_with_edits,
is_claude_model_target,
merge_anthropic_beta_headers,
tools_without_custom_field,
@ -413,7 +413,7 @@ class AnthropicCountTokensConfig:
drop_custom=drop_params,
),
),
("context_management", anthropic_context_management(context_management)),
("context_management", context_management_with_edits(context_management)),
)
if value is not None
}

View file

@ -3,7 +3,7 @@ from typing import Final
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
messages_carry_output_config,
@ -171,28 +171,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
if existing_beta:
beta_values.update(b.strip() for b in existing_beta.split(","))
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits: Final = context_management_param.get("edits", [])
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_values.update(context_management_betas(optional_params.get("context_management")))
# Check for web search tool
for tool in tools:

View file

@ -8,7 +8,7 @@ from typing import Final
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
from fastapi.responses import JSONResponse
from pydantic import JsonValue, ValidationError
from pydantic import JsonValue, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
@ -55,6 +55,7 @@ from litellm.proxy.litellm_pre_call_utils import (
should_auto_drop_params_for_agentic_cli,
)
from litellm.router import Router
from litellm.types.utils import TokenCountResponse
router: Final = APIRouter()
@ -355,6 +356,15 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params:
}
_PROVIDER_COUNT_REPLY: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
def _provider_context_management(response: TokenCountResponse) -> Mapping[str, object]:
original: Final = _PROVIDER_COUNT_REPLY.validate_python(response.original_response)
context_management: Final = original.get("context_management") if original is not None else None
return {} if context_management is None else {"context_management": context_management}
@router.post(
"/v1/messages/count_tokens",
tags=["[beta] Anthropic Messages Token Counting"],
@ -434,7 +444,11 @@ async def count_tokens(
log_llm_api_exception(e, litellm_call_id)
return _anthropic_error_json_response(proxy_exception_from_route_error(e), request)
else:
return {"input_tokens": token_response.total_tokens, **count_source_fields(token_response)}
return {
"input_tokens": token_response.total_tokens,
**_provider_context_management(token_response),
**count_source_fields(token_response),
}
@router.post(

View file

@ -991,3 +991,31 @@ async def test_counter_forwards_context_management_with_the_beta_inference_sends
assert result is not None and result.total_tokens == 259
assert TypeAdapter(dict[str, JsonValue]).validate_json(sent.content)["context_management"] == forwarded
assert (sent_beta in betas, unsent_beta in betas) == (True, False)
@pytest.mark.asyncio
@pytest.mark.usefixtures("httpx_transport_clients")
@pytest.mark.parametrize("context_management", ({}, {"edits": []}, []), ids=("empty_object", "empty_edits", "empty_list"))
async def test_counter_leaves_out_context_management_without_edits(context_management: JsonValue) -> None:
"""Anthropic's count endpoint rejects a context_management with no edits, which inference accepts, so the count
leaves it out and adds no context-management beta."""
with respx.mock:
route: Final = respx.post("https://gateway.example/v1/messages/count_tokens").mock(
return_value=httpx.Response(200, json={"input_tokens": 11})
)
result: Final = await AnthropicTokenCounter().count_tokens(
model_to_use="claude-opus-5-5",
messages=[{"role": "user", "content": "Hello"}],
contents=None,
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
context_management=CountContextManagement(spec=context_management, drop_invalid_edits=False),
)
sent: Final = route.calls.last.request
betas: Final = frozenset(sent.headers.get("anthropic-beta", "").split(","))
assert result is not None and result.total_tokens == 11
assert "context_management" not in TypeAdapter(dict[str, JsonValue]).validate_json(sent.content)
assert not betas & {
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
}

View file

@ -507,6 +507,29 @@ def test_both_compact_and_context_management_headers_added():
)
def test_openai_list_form_context_management_sends_the_compact_beta():
config: Final = VertexAIPartnerModelsAnthropicMessagesConfig()
with (
patch.object(config, "_ensure_access_token", return_value=("token", "test-project")),
patch.object(config, "get_complete_vertex_url", return_value="https://mock-url"),
):
updated_headers, _ = config.validate_anthropic_messages_environment(
headers={},
model="claude-vertex-ai-opus-4-6",
messages=[],
optional_params={"context_management": [{"type": "compaction", "compact_threshold": 150000}]},
litellm_params={
"vertex_ai_project": "test-project",
"vertex_ai_location": "us-central1",
"vertex_credentials": "{}",
},
api_base=None,
)
assert updated_headers["anthropic-beta"].split(",") == [ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value]
def test_validate_environment_always_refreshes_token_ignoring_stale_bearer():
"""Regression: stale Authorization in shared deployment extra_headers must not
skip token refresh on /v1/messages — _ensure_access_token is always called."""

View file

@ -6,6 +6,7 @@ import json
import logging
import unittest
from collections.abc import Callable, Mapping
from types import MappingProxyType
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
@ -660,7 +661,10 @@ async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_e
def _count_on_claude(
monkeypatch: pytest.MonkeyPatch, peer: Callable[[httpx.Request], httpx.Response], headers: Mapping[str, str]
monkeypatch: pytest.MonkeyPatch,
peer: Callable[[httpx.Request], httpx.Response],
headers: Mapping[str, str],
extra_body: Mapping[str, object] = MappingProxyType({}),
) -> httpx.Response:
from fastapi import FastAPI
@ -700,7 +704,7 @@ def _count_on_claude(
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth()
return TestClient(app).post(
"/v1/messages/count_tokens",
json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}]},
json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}], **extra_body},
headers=dict(headers),
)
@ -728,6 +732,24 @@ def test_count_tokens_logs_one_line_with_the_callers_call_id_model_and_count(
)
def test_count_tokens_returns_the_context_management_the_provider_applied(monkeypatch: pytest.MonkeyPatch) -> None:
applied: Final = {"original_input_tokens": 640}
response: Final = _count_on_claude(
monkeypatch,
lambda request: httpx.Response(
200, json={"input_tokens": 576, "context_management": applied}, request=request
),
{},
{"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}},
)
assert response.json() == {
"input_tokens": 576,
"context_management": applied,
"litellm_count_source": "provider",
}, response.text
def test_count_tokens_provider_rejection_logs_its_status_and_answers_the_same_error(
monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]]
) -> None: