mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(anthropic): forward count context_management only with edits, map Vertex list-form betas, and return provider context_management
This commit is contained in:
parent
a3fc1485d1
commit
4550fc7ac2
7 changed files with 107 additions and 32 deletions
|
|
@ -218,10 +218,19 @@ def anthropic_context_management(context_management: JsonValue) -> JsonValue:
|
|||
return context_management if edits is None else _JSON_VALUE.validate_python(edits)
|
||||
|
||||
|
||||
def _context_management_edits(spec: JsonValue) -> Sequence[object]:
|
||||
mapping: Final = as_str_mapping(spec)
|
||||
return (validated_object_list(mapping.get("edits")) if mapping is not None else None) or ()
|
||||
|
||||
|
||||
def context_management_with_edits(context_management: JsonValue) -> JsonValue:
|
||||
spec: Final = anthropic_context_management(context_management)
|
||||
return spec if _context_management_edits(spec) else None
|
||||
|
||||
|
||||
def context_management_betas(context_management: JsonValue) -> tuple[str, ...]:
|
||||
spec: Final = as_str_mapping(anthropic_context_management(context_management))
|
||||
edits: Final = validated_object_list(spec.get("edits")) if spec is not None else None
|
||||
edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits or ()) if edit is not None)
|
||||
edits: Final = _context_management_edits(anthropic_context_management(context_management))
|
||||
edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits) if edit is not None)
|
||||
return tuple(
|
||||
beta
|
||||
for beta, used in (
|
||||
|
|
|
|||
|
|
@ -27,9 +27,9 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
|
|||
)
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
anthropic_context_management,
|
||||
carries_signed_compaction_block,
|
||||
context_management_betas,
|
||||
context_management_with_edits,
|
||||
is_claude_model_target,
|
||||
merge_anthropic_beta_headers,
|
||||
tools_without_custom_field,
|
||||
|
|
@ -413,7 +413,7 @@ class AnthropicCountTokensConfig:
|
|||
drop_custom=drop_params,
|
||||
),
|
||||
),
|
||||
("context_management", anthropic_context_management(context_management)),
|
||||
("context_management", context_management_with_edits(context_management)),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from typing import Final
|
|||
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
messages_carry_output_config,
|
||||
|
|
@ -171,28 +171,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
|
|||
if existing_beta:
|
||||
beta_values.update(b.strip() for b in existing_beta.split(","))
|
||||
|
||||
# Check for context management
|
||||
context_management_param: Final = optional_params.get("context_management")
|
||||
if context_management_param is not None:
|
||||
# Check edits array for compact_20260112 type
|
||||
edits: Final = context_management_param.get("edits", [])
|
||||
has_compact = False
|
||||
has_other = False
|
||||
|
||||
for edit in edits:
|
||||
edit_type = edit.get("type", "")
|
||||
if edit_type == "compact_20260112":
|
||||
has_compact = True
|
||||
else:
|
||||
has_other = True
|
||||
|
||||
# Add compact header if any compact edits exist
|
||||
if has_compact:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
|
||||
|
||||
# Add context management header if any other edits exist
|
||||
if has_other:
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
|
||||
beta_values.update(context_management_betas(optional_params.get("context_management")))
|
||||
|
||||
# Check for web search tool
|
||||
for tool in tools:
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from typing import Final
|
|||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import JsonValue, ValidationError
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -55,6 +55,7 @@ from litellm.proxy.litellm_pre_call_utils import (
|
|||
should_auto_drop_params_for_agentic_cli,
|
||||
)
|
||||
from litellm.router import Router
|
||||
from litellm.types.utils import TokenCountResponse
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
||||
|
|
@ -355,6 +356,15 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params:
|
|||
}
|
||||
|
||||
|
||||
_PROVIDER_COUNT_REPLY: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
|
||||
|
||||
def _provider_context_management(response: TokenCountResponse) -> Mapping[str, object]:
|
||||
original: Final = _PROVIDER_COUNT_REPLY.validate_python(response.original_response)
|
||||
context_management: Final = original.get("context_management") if original is not None else None
|
||||
return {} if context_management is None else {"context_management": context_management}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/messages/count_tokens",
|
||||
tags=["[beta] Anthropic Messages Token Counting"],
|
||||
|
|
@ -434,7 +444,11 @@ async def count_tokens(
|
|||
log_llm_api_exception(e, litellm_call_id)
|
||||
return _anthropic_error_json_response(proxy_exception_from_route_error(e), request)
|
||||
else:
|
||||
return {"input_tokens": token_response.total_tokens, **count_source_fields(token_response)}
|
||||
return {
|
||||
"input_tokens": token_response.total_tokens,
|
||||
**_provider_context_management(token_response),
|
||||
**count_source_fields(token_response),
|
||||
}
|
||||
|
||||
|
||||
@router.post(
|
||||
|
|
|
|||
|
|
@ -991,3 +991,31 @@ async def test_counter_forwards_context_management_with_the_beta_inference_sends
|
|||
assert result is not None and result.total_tokens == 259
|
||||
assert TypeAdapter(dict[str, JsonValue]).validate_json(sent.content)["context_management"] == forwarded
|
||||
assert (sent_beta in betas, unsent_beta in betas) == (True, False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("httpx_transport_clients")
|
||||
@pytest.mark.parametrize("context_management", ({}, {"edits": []}, []), ids=("empty_object", "empty_edits", "empty_list"))
|
||||
async def test_counter_leaves_out_context_management_without_edits(context_management: JsonValue) -> None:
|
||||
"""Anthropic's count endpoint rejects a context_management with no edits, which inference accepts, so the count
|
||||
leaves it out and adds no context-management beta."""
|
||||
with respx.mock:
|
||||
route: Final = respx.post("https://gateway.example/v1/messages/count_tokens").mock(
|
||||
return_value=httpx.Response(200, json={"input_tokens": 11})
|
||||
)
|
||||
result: Final = await AnthropicTokenCounter().count_tokens(
|
||||
model_to_use="claude-opus-5-5",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}},
|
||||
context_management=CountContextManagement(spec=context_management, drop_invalid_edits=False),
|
||||
)
|
||||
|
||||
sent: Final = route.calls.last.request
|
||||
betas: Final = frozenset(sent.headers.get("anthropic-beta", "").split(","))
|
||||
assert result is not None and result.total_tokens == 11
|
||||
assert "context_management" not in TypeAdapter(dict[str, JsonValue]).validate_json(sent.content)
|
||||
assert not betas & {
|
||||
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
|
||||
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -507,6 +507,29 @@ def test_both_compact_and_context_management_headers_added():
|
|||
)
|
||||
|
||||
|
||||
def test_openai_list_form_context_management_sends_the_compact_beta():
|
||||
config: Final = VertexAIPartnerModelsAnthropicMessagesConfig()
|
||||
|
||||
with (
|
||||
patch.object(config, "_ensure_access_token", return_value=("token", "test-project")),
|
||||
patch.object(config, "get_complete_vertex_url", return_value="https://mock-url"),
|
||||
):
|
||||
updated_headers, _ = config.validate_anthropic_messages_environment(
|
||||
headers={},
|
||||
model="claude-vertex-ai-opus-4-6",
|
||||
messages=[],
|
||||
optional_params={"context_management": [{"type": "compaction", "compact_threshold": 150000}]},
|
||||
litellm_params={
|
||||
"vertex_ai_project": "test-project",
|
||||
"vertex_ai_location": "us-central1",
|
||||
"vertex_credentials": "{}",
|
||||
},
|
||||
api_base=None,
|
||||
)
|
||||
|
||||
assert updated_headers["anthropic-beta"].split(",") == [ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value]
|
||||
|
||||
|
||||
def test_validate_environment_always_refreshes_token_ignoring_stale_bearer():
|
||||
"""Regression: stale Authorization in shared deployment extra_headers must not
|
||||
skip token refresh on /v1/messages — _ensure_access_token is always called."""
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import json
|
|||
import logging
|
||||
import unittest
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -660,7 +661,10 @@ async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_e
|
|||
|
||||
|
||||
def _count_on_claude(
|
||||
monkeypatch: pytest.MonkeyPatch, peer: Callable[[httpx.Request], httpx.Response], headers: Mapping[str, str]
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
peer: Callable[[httpx.Request], httpx.Response],
|
||||
headers: Mapping[str, str],
|
||||
extra_body: Mapping[str, object] = MappingProxyType({}),
|
||||
) -> httpx.Response:
|
||||
from fastapi import FastAPI
|
||||
|
||||
|
|
@ -700,7 +704,7 @@ def _count_on_claude(
|
|||
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth()
|
||||
return TestClient(app).post(
|
||||
"/v1/messages/count_tokens",
|
||||
json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}]},
|
||||
json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}], **extra_body},
|
||||
headers=dict(headers),
|
||||
)
|
||||
|
||||
|
|
@ -728,6 +732,24 @@ def test_count_tokens_logs_one_line_with_the_callers_call_id_model_and_count(
|
|||
)
|
||||
|
||||
|
||||
def test_count_tokens_returns_the_context_management_the_provider_applied(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
applied: Final = {"original_input_tokens": 640}
|
||||
response: Final = _count_on_claude(
|
||||
monkeypatch,
|
||||
lambda request: httpx.Response(
|
||||
200, json={"input_tokens": 576, "context_management": applied}, request=request
|
||||
),
|
||||
{},
|
||||
{"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}},
|
||||
)
|
||||
|
||||
assert response.json() == {
|
||||
"input_tokens": 576,
|
||||
"context_management": applied,
|
||||
"litellm_count_source": "provider",
|
||||
}, response.text
|
||||
|
||||
|
||||
def test_count_tokens_provider_rejection_logs_its_status_and_answers_the_same_error(
|
||||
monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]]
|
||||
) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue