scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-03 02:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 80

fix(anthropic): normalize [1m] models before capability lookups

Browse source 
Review follow-ups for the context-1m change

The chat transform now treats a [1m] request exactly like its base model, so
model-info lookups (default max_tokens, mid-conversation system support,
thinking and speed handling) no longer miss on the suffixed name. Before this,
anthropic/claude-sonnet-4-6[1m] silently fell back to a 4096 max_tokens default

Detection now reads only the model string. The _original_model stash was
never populated once proxy-level stripping was dropped, so those lookups and
the duplicate injection in update_headers_with_optional_anthropic_beta are
removed. The beta value lives in ANTHROPIC_BETA_HEADER_VALUES

Tests move to the mirrored unit test files and cover litellm.acompletion and
Router.aanthropic_messages end to end with the HTTP layer mocked

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
hx 2026-09-27 18:05:43 +08:00
parent d5f6274c1e
commit c48bd40f1b
8 changed files with 178 additions and 224 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
litellm/llms/anthropic/chat/transformation.py
View file

@ -99,8 +99,8 @@ from litellm.utils import (
from ..common_utils import (
AnthropicError,
AnthropicModelInfo,
context_1m_requested,
eager_input_streaming_flag,
has_context_1m_suffix,
process_anthropic_headers,
requires_native_compaction_beta,
strip_advisor_blocks_from_messages,
@ -1781,17 +1781,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
return tools
def _maybe_add_context_1m_beta(
self,
headers: dict,
*,
model: str,
optional_params: dict,
litellm_params: Mapping[str, object] | None,
) -> None:
if context_1m_requested(model=model, optional_params=optional_params, litellm_params=litellm_params):
self._ensure_beta_header(headers, "context-1m-2025-08-07")
def _ensure_beta_header(self, headers: dict[str, str], beta_value: str) -> None:
"""
Ensure a beta header value is present in the anthropic-beta header.
@ -1850,12 +1839,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
def update_headers_with_optional_anthropic_beta(
self,
headers: dict,
optional_params: dict,
messages: Sequence[object] = (),
litellm_params: Mapping[str, object] | None = None,
model: str = "",
self, headers: dict, optional_params: dict, messages: Sequence[object] = ()
) -> dict:
"""Update headers with optional anthropic beta."""
@ -1868,10 +1852,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if requires_native_compaction_beta(self._resolved_provider, optional_params, messages):
self._ensure_beta_header(headers, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
self._maybe_add_context_1m_beta(
headers, model=model, optional_params=optional_params, litellm_params=litellm_params
)
_tools: Final = optional_params.get("tools", [])
for tool in _tools:
if tool.get("type", None) and tool.get("type").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_FETCH.value):
@ -1926,6 +1906,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"""
Translate messages to anthropic format.
"""
if has_context_1m_suffix(model):
return AnthropicConfig.transform_request(
self,
model=strip_context_1m_suffix(model),
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
headers=headers,
)
## VALIDATE REQUEST
"""Anthropic requires ``tools`` when messages include tool blocks; LiteLLM injects a dummy tool if omitted (no ``modify_params`` needed)."""
from litellm.litellm_core_utils.prompt_templates.factory import (
@ -2026,11 +2015,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
) # don't use verbose_logger.exception, if exception is raised
self.update_headers_with_optional_anthropic_beta(
headers=headers,
optional_params=optional_params,
messages=anthropic_messages,
litellm_params=litellm_params,
model=model,
headers=headers, optional_params=optional_params, messages=anthropic_messages
)
## Auto-strip advisor blocks from history if advisor tool is absent.
@ -2098,7 +2083,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
data: Final = {
"model": strip_context_1m_suffix(model),
"model": model,
"messages": anthropic_messages,
**optional_params,
}

49
litellm/llms/anthropic/common_utils.py
View file

@ -30,6 +30,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_OAUTH_BETA_HEADER,
ANTHROPIC_OAUTH_TOKEN_PREFIX,
@ -74,43 +75,14 @@ _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)")
_CONTEXT_1M_SUFFIX: Final = re.compile(r"\[1m\]$", flags=re.IGNORECASE)
def model_has_context_1m_suffix(model: object) -> bool:
return isinstance(model, str) and _CONTEXT_1M_SUFFIX.search(model) is not None
def has_context_1m_suffix(model: str) -> bool:
return _CONTEXT_1M_SUFFIX.search(model) is not None
def strip_context_1m_suffix(model: str) -> str:
return _CONTEXT_1M_SUFFIX.sub("", model)
def _original_model_from(params: object) -> object:
if not isinstance(params, Mapping):
return None
return params.get("_original_model")
def context_1m_requested(
*,
model: object = "",
optional_params: object = None,
litellm_params: object = None,
) -> bool:
return any(
model_has_context_1m_suffix(candidate)
for candidate in (
model,
_original_model_from(optional_params),
_original_model_from(litellm_params),
)
)
_CONTEXT_1M_BETA: Final = "context-1m-2025-08-07"
def context_1m_beta_values(supported: bool) -> frozenset[str]:
return frozenset((_CONTEXT_1M_BETA,)) if supported else frozenset()
_CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
_CLAUDE_CODE_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
_CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
@ -955,9 +927,11 @@ class AnthropicModelInfo(BaseLLMModelInfo):
container_with_skills_used: bool = False,
api_base: str | None = None,
use_bearer_for_custom_base: bool = False,
context_1m_supported: bool = False,
context_1m_requested: bool = False,
) -> dict:
betas: Final = set(context_1m_beta_values(context_1m_supported))
betas: Final[set[str]] = set()
if context_1m_requested:
betas.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_1M_2025_08_07.value)
# Anthropic no longer requires the prompt-caching beta header
# Prompt caching now works automatically when cache_control is used in messages
# Reference: https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching
@ -1013,8 +987,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
if is_vertex_request is True:
# Vertex AI requires web search beta header for web search to work
if web_search_tool_used:
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
elif len(betas) > 0:
headers["anthropic-beta"] = ",".join(betas)
@ -1066,14 +1038,9 @@ class AnthropicModelInfo(BaseLLMModelInfo):
user_anthropic_beta_headers: Final = self._get_user_anthropic_beta_headers(
anthropic_beta_header=headers.get("anthropic-beta")
)
context_1m_supported: Final = context_1m_requested(
model=model,
optional_params=optional_params,
litellm_params=litellm_params,
)
anthropic_headers: Final = self.get_anthropic_headers(
computer_tool_used=computer_tool_used,
context_1m_supported=context_1m_supported,
context_1m_requested=has_context_1m_suffix(model),
prompt_caching_set=prompt_caching_set,
pdf_used=pdf_used,
api_key=api_key,

17
litellm/llms/anthropic/pass_through/messages/transformation.py
View file

@ -23,8 +23,7 @@ from litellm.types.router import GenericLiteLLMParams
from ...common_utils import (
AnthropicError,
AnthropicModelInfo,
context_1m_beta_values,
context_1m_requested,
has_context_1m_suffix,
optionally_handle_anthropic_oauth,
requires_native_compaction_beta,
strip_advisor_blocks_from_messages,
@ -281,7 +280,6 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
optional_params=optional_params,
messages=messages,
model=model,
litellm_params=litellm_params,
)
return headers, api_base
@ -617,7 +615,6 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
custom_llm_provider: str = "anthropic",
messages: Sequence[object] = (),
model: str = "",
litellm_params: object = None,
) -> dict:
"""
Auto-inject anthropic-beta headers based on features used.
@ -628,15 +625,16 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
- output_format: adds 'structured-outputs-2025-11-13'
- speed: adds 'fast-mode-2026-02-01'
- a message carrying output_config: adds 'per-turn-control-2026-07-01'
- [1m] suffix: adds 'context-1m-2025-08-07'
- [1m] model suffix: adds 'context-1m-2025-08-07'
Args:
headers: Request headers dict
optional_params: Optional parameters including tools, context_management, output_format, speed
custom_llm_provider: Provider name for looking up correct tool search header
messages: Request messages, scanned for per-message output_config
model: Requested model name, checked for the [1m] suffix
"""
beta_values: Final[set] = set()
beta_values: Final[set[str]] = set()
existing_beta: Final = tuple(
piece.strip()
@ -650,11 +648,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
beta_values.update(
context_1m_beta_values(
context_1m_requested(model=model, optional_params=optional_params, litellm_params=litellm_params)
)
)
if has_context_1m_suffix(model):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_1M_2025_08_07.value)
# Check for context management
context_management_param: Final = optional_params.get("context_management")

1
litellm/types/llms/anthropic.py
View file

@ -754,6 +754,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum):
WEB_FETCH_2025_09_10 = "web-fetch-2025-09-10"
WEB_SEARCH_2025_03_05 = "web-search-2025-03-05"
CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27"
CONTEXT_1M_2025_08_07 = "context-1m-2025-08-07"
COMPACT_2026_01_12 = "compact-2026-01-12"
COMPACT_2026_09_04 = "compact-2026-09-04"
STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13"

55
tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -6798,3 +6798,58 @@ def test_chat_dummy_tool_result_for_an_orphaned_tool_call_replays_a_byte_identic
_assert_prefix_stable(requests)
assert [m["role"] for m in requests[0]["messages"]] == ["user", "assistant", "user"]
assert requests[0]["messages"][2]["content"][0]["type"] == "tool_result"
@pytest.mark.parametrize("optional_params", [{}, {"thinking": {"type": "enabled", "budget_tokens": 2048}}])
def test_transform_request_treats_context_1m_model_as_its_base_model(optional_params: dict[str, object]) -> None:
def transform(model: str) -> dict[str, object]:
return AnthropicConfig().transform_request(
model=model,
messages=[{"role": "user", "content": "hi"}],
optional_params=copy.deepcopy(optional_params),
litellm_params={},
headers={},
)
assert transform("claude-sonnet-4-6[1m]") == transform("claude-sonnet-4-6")
@pytest.mark.parametrize(
"model,expected_betas",
[
("anthropic/claude-sonnet-4-6[1m]", {"context-1m-2025-08-07", "interleaved-thinking-2025-05-14"}),
("anthropic/claude-sonnet-4-6", {"interleaved-thinking-2025-05-14"}),
],
)
async def test_acompletion_context_1m_suffix_sends_base_model_with_beta(
model: str,
expected_betas: set[str],
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True")
monkeypatch.setattr(litellm.anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None)
route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").respond(
200,
json={
"id": "msg_context_1m",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-6",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1},
},
)
await litellm.acompletion(
model=model,
messages=[{"role": "user", "content": "hi"}],
api_key="test",
extra_headers={"anthropic-beta": "interleaved-thinking-2025-05-14"},
)
request: Final = route.calls.last.request
assert json.loads(request.content)["model"] == "claude-sonnet-4-6"
assert {beta.strip() for beta in request.headers["anthropic-beta"].split(",")} == expected_betas

57
tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_context_1m.py Normal file
View file

@ -0,0 +1,57 @@
import json
from typing import Final
import pytest
import respx
import litellm
from litellm import Router
@pytest.mark.parametrize(
"deployment_model,expected_betas",
[
("anthropic/claude-sonnet-4-6[1m]", {"context-1m-2025-08-07", "interleaved-thinking-2025-05-14"}),
("anthropic/claude-sonnet-4-6", {"interleaved-thinking-2025-05-14"}),
],
)
async def test_router_messages_context_1m_deployment_sends_base_model_with_beta(
deployment_model: str,
expected_betas: set[str],
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True")
monkeypatch.setattr(litellm.anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None)
route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").respond(
200,
json={
"id": "msg_context_1m",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-6",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1},
},
)
router: Final = Router(
model_list=[
{
"model_name": "claude-sonnet-4-6[1m]",
"litellm_params": {"model": deployment_model, "api_key": "test"},
}
]
)
await router.aanthropic_messages(
model="claude-sonnet-4-6[1m]",
messages=[{"role": "user", "content": "hi"}],
max_tokens=16,
extra_headers={"anthropic-beta": "interleaved-thinking-2025-05-14"},
)
request: Final = route.calls.last.request
assert json.loads(request.content)["model"] == "claude-sonnet-4-6"
assert {beta.strip() for beta in request.headers["anthropic-beta"].split(",")} == expected_betas

38
tests/unit/llms/anthropic/test_anthropic_common_utils.py
View file

@ -2345,3 +2345,41 @@ class TestMalformedContentListItems:
api_key=FAKE_REGULAR_KEY,
max_tokens=5,
)
@pytest.mark.parametrize(
"model,base_model",
[
("claude-sonnet-4-6[1m]", "claude-sonnet-4-6"),
("claude-sonnet-4-6[1M]", "claude-sonnet-4-6"),
("claude-sonnet-4-6", "claude-sonnet-4-6"),
("claude-[1m]-sonnet-4-6", "claude-[1m]-sonnet-4-6"),
("claude-sonnet-4-6[1m][1m]", "claude-sonnet-4-6[1m]"),
("claude-sonnet-4-6[200k]", "claude-sonnet-4-6[200k]"),
],
)
def test_context_1m_suffix_is_only_a_trailing_marker(model: str, base_model: str) -> None:
from litellm.llms.anthropic.common_utils import has_context_1m_suffix, strip_context_1m_suffix
assert strip_context_1m_suffix(model) == base_model
assert has_context_1m_suffix(model) is (model != base_model)
@pytest.mark.parametrize("model,expects_context_1m", [("claude-sonnet-4-6[1m]", True), ("claude-sonnet-4-6", False)])
def test_validate_environment_adds_context_1m_beta_only_for_suffixed_model(
model: str, expects_context_1m: bool
) -> None:
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
headers: Final = AnthropicModelInfo().validate_environment(
headers={"anthropic-beta": "interleaved-thinking-2025-05-14"},
model=model,
messages=[{"role": "user", "content": "hi"}],
optional_params={},
litellm_params={},
api_key=FAKE_REGULAR_KEY,
)
betas: Final = {beta.strip() for beta in headers["anthropic-beta"].split(",")}
assert ("context-1m-2025-08-07" in betas) is expects_context_1m
assert "interleaved-thinking-2025-05-14" in betas

144
tests/unit/llms/anthropic/test_context_1m_header.py
View file

@ -1,144 +0,0 @@
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.common_utils import (
context_1m_requested,
strip_context_1m_suffix,
)
from litellm.llms.anthropic.pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
class TestAnthropicContext1mBetaHeader:
def test_strip_context_1m_suffix_is_pure(self):
assert strip_context_1m_suffix("claude-sonnet-4-6[1m]") == "claude-sonnet-4-6"
assert strip_context_1m_suffix("claude-sonnet-4-6[1M]") == "claude-sonnet-4-6"
assert strip_context_1m_suffix("claude-sonnet-4-6") == "claude-sonnet-4-6"
assert strip_context_1m_suffix("gpt-4o[1m]") == "gpt-4o"
def test_context_1m_requested_from_model_or_stashed_original(self):
assert context_1m_requested(model="claude-sonnet-4-6[1m]")
assert context_1m_requested(
model="claude-sonnet-4-6",
litellm_params={"_original_model": "claude-sonnet-4-6[1M]"},
)
assert not context_1m_requested(model="claude-sonnet-4-6")
assert not context_1m_requested(model="gpt-4o")
def test_get_anthropic_headers_with_context_1m(self):
headers = AnthropicConfig().get_anthropic_headers(api_key="sk-test", context_1m_supported=True)
assert "context-1m-2025-08-07" in headers.get("anthropic-beta", "")
def test_get_anthropic_headers_without_context_1m(self):
headers = AnthropicConfig().get_anthropic_headers(api_key="sk-test", context_1m_supported=False)
assert "context-1m-2025-08-07" not in headers.get("anthropic-beta", "")
def test_validate_environment_adds_context_1m_for_suffixed_model(self):
headers = AnthropicConfig().validate_environment(
headers={},
model="claude-sonnet-4-6[1m]",
messages=[{"role": "user", "content": "hi"}],
optional_params={},
litellm_params={},
api_key="sk-test",
)
assert "context-1m-2025-08-07" in headers.get("anthropic-beta", "")
def test_validate_environment_no_context_1m_without_suffix(self):
headers = AnthropicConfig().validate_environment(
headers={},
model="claude-sonnet-4-6",
messages=[{"role": "user", "content": "hi"}],
optional_params={},
litellm_params={},
api_key="sk-test",
)
assert "context-1m-2025-08-07" not in headers.get("anthropic-beta", "")
def test_validate_environment_uses_original_model_from_litellm_params(self):
headers = AnthropicConfig().validate_environment(
headers={},
model="claude-sonnet-4-6",
messages=[{"role": "user", "content": "hi"}],
optional_params={},
litellm_params={"_original_model": "claude-sonnet-4-6[1M]"},
api_key="sk-test",
)
assert "context-1m-2025-08-07" in headers.get("anthropic-beta", "")
def test_update_headers_adds_context_1m_with_original_model(self):
headers = AnthropicConfig().update_headers_with_optional_anthropic_beta(
headers={},
optional_params={"_original_model": "claude-sonnet-4-6[1m]"},
)
assert "context-1m-2025-08-07" in headers.get("anthropic-beta", "")
def test_update_headers_no_context_1m_without_original_model(self):
headers = AnthropicConfig().update_headers_with_optional_anthropic_beta(
headers={},
optional_params={},
)
assert "context-1m-2025-08-07" not in headers.get("anthropic-beta", "")
def test_pass_through_adds_context_1m(self):
headers = AnthropicMessagesConfig._update_headers_with_anthropic_beta(
headers={},
optional_params={"_original_model": "claude-sonnet-4-6[1m]"},
)
assert "context-1m-2025-08-07" in headers.get("anthropic-beta", "")
def test_pass_through_no_context_1m_without_suffix(self):
headers = AnthropicMessagesConfig._update_headers_with_anthropic_beta(
headers={},
optional_params={},
)
assert "context-1m-2025-08-07" not in headers.get("anthropic-beta", "")
def test_pass_through_adds_context_1m_from_model_without_original_model(self):
headers, _ = AnthropicMessagesConfig().validate_anthropic_messages_environment(
headers={},
model="claude-sonnet-4-6[1m]",
messages=[{"role": "user", "content": "hi"}],
optional_params={},
litellm_params={},
api_key="sk-test",
)
assert "context-1m-2025-08-07" in headers.get("anthropic-beta", "")
def test_pass_through_transform_strips_1m_from_upstream_model(self):
result = AnthropicMessagesConfig().transform_anthropic_messages_request(
model="claude-sonnet-4-6[1m]",
messages=[{"role": "user", "content": "hi"}],
anthropic_messages_optional_request_params={"max_tokens": 16},
litellm_params={},
headers={},
)
assert result["model"] == "claude-sonnet-4-6"
def test_chat_transform_request_strips_1m_and_injects_header(self):
headers = {}
result = AnthropicConfig().transform_request(
model="claude-sonnet-4-6[1m]",
messages=[{"role": "user", "content": "hi"}],
optional_params={"max_tokens": 16},
litellm_params={},
headers=headers,
)
assert result["model"] == "claude-sonnet-4-6"
assert "context-1m-2025-08-07" in headers.get("anthropic-beta", "")
def test_chat_transform_request_uppercase_suffix_strips_and_injects_header(self):
headers = {}
result = AnthropicConfig().transform_request(
model="claude-sonnet-4-6[1M]",
messages=[{"role": "user", "content": "hi"}],
optional_params={"max_tokens": 16},
litellm_params={},
headers=headers,
)
assert result["model"] == "claude-sonnet-4-6"
assert "context-1m-2025-08-07" in headers.get("anthropic-beta", "")
def test_proxy_does_not_globally_strip_non_anthropic_suffix(self):
from litellm.proxy import route_llm_request
assert not hasattr(route_llm_request, "stash_and_strip_context_1m_model_suffix")