mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 4b1d761d14 into 523b87ec71
This commit is contained in:
commit
ef61157179
7 changed files with 218 additions and 3 deletions
|
|
@ -133,11 +133,13 @@ from ..common_utils import (
|
|||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
eager_input_streaming_flag,
|
||||
has_context_1m_suffix,
|
||||
process_anthropic_headers,
|
||||
refusal_params,
|
||||
requires_native_compaction_beta,
|
||||
stop_reason_provider_specific_fields,
|
||||
strip_advisor_blocks_from_messages,
|
||||
strip_context_1m_suffix,
|
||||
tools_marked_for_a_tool_config_point,
|
||||
)
|
||||
|
||||
|
|
@ -2370,6 +2372,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
"""
|
||||
Translate messages to anthropic format.
|
||||
"""
|
||||
if has_context_1m_suffix(model):
|
||||
return AnthropicConfig.transform_request(
|
||||
self,
|
||||
model=strip_context_1m_suffix(model),
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
## VALIDATE REQUEST
|
||||
"""Anthropic requires ``tools`` when messages include tool blocks; LiteLLM injects a dummy tool if omitted (no ``modify_params`` needed)."""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
|
|
|
|||
|
|
@ -121,6 +121,17 @@ _BEDROCK_VERSION_SUFFIX_RE: Final = re.compile(r"-v\d+(?::\d+)?$")
|
|||
_INFERENCE_PROFILE_MINOR_RE: Final = re.compile(r":\d+$")
|
||||
_DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$")
|
||||
_DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)")
|
||||
_CONTEXT_1M_SUFFIX: Final = re.compile(r"\[1m\]$", flags=re.IGNORECASE)
|
||||
|
||||
|
||||
def has_context_1m_suffix(model: str) -> bool:
|
||||
return _CONTEXT_1M_SUFFIX.search(model) is not None
|
||||
|
||||
|
||||
def strip_context_1m_suffix(model: str) -> str:
|
||||
return _CONTEXT_1M_SUFFIX.sub("", model)
|
||||
|
||||
|
||||
_CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:"
|
||||
OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
|
||||
OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
|
||||
|
|
@ -1429,6 +1440,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
thinking: AnthropicThinkingParam | None = None,
|
||||
is_mid_conversation_tool_change_used: bool = False,
|
||||
wif_minted: bool = False,
|
||||
context_1m_requested: bool = False,
|
||||
) -> dict:
|
||||
betas: Final = set()
|
||||
# Anthropic no longer requires the prompt-caching beta header
|
||||
|
|
@ -1487,6 +1499,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
|
||||
all_betas: Final = betas.union(
|
||||
self.thinking_betas(thinking),
|
||||
(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_1M_2025_08_07.value,) if context_1m_requested else (),
|
||||
(ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) if is_mid_conversation_tool_change_used else (),
|
||||
)
|
||||
|
||||
|
|
@ -1565,6 +1578,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
)
|
||||
anthropic_headers: Final = self.get_anthropic_headers(
|
||||
computer_tool_used=computer_tool_used,
|
||||
context_1m_requested=has_context_1m_suffix(model),
|
||||
prompt_caching_set=prompt_caching_set,
|
||||
pdf_used=pdf_used,
|
||||
api_key=resolved_api_key,
|
||||
|
|
|
|||
|
|
@ -46,11 +46,13 @@ from ...common_utils import (
|
|||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
context_management_betas,
|
||||
has_context_1m_suffix,
|
||||
is_claude_model_target,
|
||||
merge_anthropic_beta_headers,
|
||||
optionally_handle_anthropic_oauth,
|
||||
requires_native_compaction_beta,
|
||||
strip_advisor_blocks_from_messages,
|
||||
strip_context_1m_suffix,
|
||||
strip_encrypted_reasoning_blocks_from_anthropic_messages,
|
||||
tools_without_custom_field,
|
||||
)
|
||||
|
|
@ -374,7 +376,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
model=model,
|
||||
),
|
||||
)
|
||||
return self.finalize_messages_headers(headers, optional_params, messages), api_base
|
||||
return self.finalize_messages_headers(headers, optional_params, messages, model=model), api_base
|
||||
|
||||
async def avalidate_anthropic_messages_environment(
|
||||
self,
|
||||
|
|
@ -414,7 +416,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
model=model,
|
||||
),
|
||||
)
|
||||
return self.finalize_messages_headers(oauth_headers, optional_params, messages), api_base
|
||||
return self.finalize_messages_headers(oauth_headers, optional_params, messages, model=model), api_base
|
||||
|
||||
def _require_auth_header(self, auth_header: Mapping[str, str] | None, model: str) -> Mapping[str, str]:
|
||||
if auth_header is None:
|
||||
|
|
@ -453,6 +455,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
headers: dict, # mutable-ok: out-param
|
||||
optional_params: dict, # mutable-ok: out-param
|
||||
messages: list[object], # mutable-ok: mirrors the validate_anthropic_messages_environment contract
|
||||
model: str = "",
|
||||
) -> dict: # mutable-ok: out-param
|
||||
if "anthropic-version" not in headers:
|
||||
headers["anthropic-version"] = DEFAULT_ANTHROPIC_API_VERSION
|
||||
|
|
@ -462,6 +465,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
headers=headers,
|
||||
optional_params=optional_params,
|
||||
messages=messages,
|
||||
model=model,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -792,7 +796,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
)
|
||||
),
|
||||
max_tokens=max_tokens,
|
||||
model=model,
|
||||
model=strip_context_1m_suffix(model),
|
||||
**anthropic_messages_optional_request_params,
|
||||
)
|
||||
request: Final = (
|
||||
|
|
@ -853,6 +857,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
messages: Sequence[object] = (),
|
||||
beta_map_provider: str | None = None,
|
||||
drop_params: bool = False,
|
||||
model: str = "",
|
||||
) -> dict:
|
||||
"""
|
||||
Auto-inject anthropic-beta headers based on features used.
|
||||
|
|
@ -863,6 +868,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
- output_format: adds 'structured-outputs-2025-11-13'
|
||||
- speed: adds 'fast-mode-2026-02-01'
|
||||
- a message carrying output_config: adds 'per-turn-control-2026-07-01'
|
||||
- [1m] model suffix: adds 'context-1m-2025-08-07'
|
||||
|
||||
Args:
|
||||
headers: Request headers dict
|
||||
|
|
@ -871,6 +877,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
messages: Request messages, scanned for per-message output_config
|
||||
beta_map_provider: Provider whose beta map column gates safeguards, defaults to custom_llm_provider
|
||||
drop_params: Whether the deployment or request set drop_params
|
||||
model: Requested model name, checked for the [1m] suffix
|
||||
"""
|
||||
beta_values: Final[set[str]] = set()
|
||||
|
||||
|
|
@ -886,6 +893,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
|
||||
|
||||
if has_context_1m_suffix(model):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_1M_2025_08_07.value)
|
||||
|
||||
beta_values.update(context_management_betas(optional_params.get("context_management")))
|
||||
|
||||
# Check for structured outputs. Anthropic's newer request shape nests
|
||||
|
|
|
|||
|
|
@ -840,6 +840,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum):
|
|||
WEB_FETCH_2025_09_10 = "web-fetch-2025-09-10"
|
||||
WEB_SEARCH_2025_03_05 = "web-search-2025-03-05"
|
||||
CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27"
|
||||
CONTEXT_1M_2025_08_07 = "context-1m-2025-08-07"
|
||||
COMPACT_2026_01_12 = "compact-2026-01-12"
|
||||
COMPACT_2026_09_04 = "compact-2026-09-04"
|
||||
INLINE_TOOLS_2026_09_15 = "inline-tools-2026-09-15"
|
||||
|
|
|
|||
|
|
@ -10650,3 +10650,62 @@ def test_claude_cache_control_cap_does_not_count_a_message_level_hint_that_list_
|
|||
|
||||
assert sorted(_cached_texts(body)) == ["system prompt", "turn 1", "turn 2", "turn 3"]
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("optional_params", [{}, {"thinking": {"type": "enabled", "budget_tokens": 2048}}])
|
||||
def test_transform_request_treats_context_1m_model_as_its_base_model(optional_params: dict[str, object]) -> None:
|
||||
def transform(model: str) -> dict[str, object]:
|
||||
return AnthropicConfig().transform_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params=copy.deepcopy(optional_params),
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert transform("claude-sonnet-5-5[1m]") == transform("claude-sonnet-5-5")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,expected_betas",
|
||||
[
|
||||
(
|
||||
"anthropic/claude-sonnet-5-5[1m]",
|
||||
{ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_1M_2025_08_07.value, "interleaved-thinking-2025-05-14"},
|
||||
),
|
||||
("anthropic/claude-sonnet-5-5", {"interleaved-thinking-2025-05-14"}),
|
||||
],
|
||||
)
|
||||
async def test_acompletion_context_1m_suffix_sends_base_model_with_beta(
|
||||
model: str,
|
||||
expected_betas: set[str],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True")
|
||||
monkeypatch.setattr(litellm.anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None)
|
||||
route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").respond(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_context_1m",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": [{"type": "text", "text": "hi"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1},
|
||||
},
|
||||
)
|
||||
|
||||
await litellm.acompletion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_key="test",
|
||||
extra_headers={"anthropic-beta": "interleaved-thinking-2025-05-14"},
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert json.loads(request.content)["model"] == "claude-sonnet-5-5"
|
||||
assert {beta.strip() for beta in request.headers["anthropic-beta"].split(",")} == expected_betas
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model,expects_context_1m", [("claude-sonnet-5-5[1m]", True), ("claude-sonnet-5-5", False)])
|
||||
def test_sync_messages_environment_adds_context_1m_beta_only_for_suffixed_model(
|
||||
model: str, expects_context_1m: bool
|
||||
) -> None:
|
||||
headers, _ = AnthropicMessagesConfig().validate_anthropic_messages_environment(
|
||||
headers={"anthropic-beta": "interleaved-thinking-2025-05-14"},
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
api_key="test",
|
||||
)
|
||||
|
||||
betas: Final = {beta.strip() for beta in headers["anthropic-beta"].split(",")}
|
||||
assert (ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_1M_2025_08_07.value in betas) is expects_context_1m
|
||||
assert "interleaved-thinking-2025-05-14" in betas
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployment_model,expected_betas",
|
||||
[
|
||||
(
|
||||
"anthropic/claude-sonnet-5-5[1m]",
|
||||
{ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_1M_2025_08_07.value, "interleaved-thinking-2025-05-14"},
|
||||
),
|
||||
("anthropic/claude-sonnet-5-5", {"interleaved-thinking-2025-05-14"}),
|
||||
],
|
||||
)
|
||||
async def test_router_messages_context_1m_deployment_sends_base_model_with_beta(
|
||||
deployment_model: str,
|
||||
expected_betas: set[str],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True")
|
||||
monkeypatch.setattr(litellm.anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None)
|
||||
route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").respond(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_context_1m",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": [{"type": "text", "text": "hi"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1},
|
||||
},
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude-sonnet-5-5[1m]",
|
||||
"litellm_params": {"model": deployment_model, "api_key": "test"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
await router.aanthropic_messages(
|
||||
model="claude-sonnet-5-5[1m]",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
max_tokens=16,
|
||||
extra_headers={"anthropic-beta": "interleaved-thinking-2025-05-14"},
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert json.loads(request.content)["model"] == "claude-sonnet-5-5"
|
||||
assert {beta.strip() for beta in request.headers["anthropic-beta"].split(",")} == expected_betas
|
||||
|
|
@ -4728,3 +4728,43 @@ def test_tools_without_strict_drops_only_the_strict_key_and_counts_the_strict_to
|
|||
assert tools_without_strict(tools) == ToolsWithoutStrict(
|
||||
tools=({"name": "a"}, {"name": "b"}, {"name": "c"}), dropped_count=1
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,base_model",
|
||||
[
|
||||
("claude-sonnet-5-5[1m]", "claude-sonnet-5-5"),
|
||||
("claude-sonnet-5-5[1M]", "claude-sonnet-5-5"),
|
||||
("claude-sonnet-5-5", "claude-sonnet-5-5"),
|
||||
("claude-[1m]-sonnet-5-5", "claude-[1m]-sonnet-5-5"),
|
||||
("claude-sonnet-5-5[1m][1m]", "claude-sonnet-5-5[1m]"),
|
||||
("claude-sonnet-5-5[200k]", "claude-sonnet-5-5[200k]"),
|
||||
],
|
||||
)
|
||||
def test_context_1m_suffix_is_only_a_trailing_marker(model: str, base_model: str) -> None:
|
||||
from litellm.llms.anthropic.common_utils import has_context_1m_suffix, strip_context_1m_suffix
|
||||
|
||||
assert strip_context_1m_suffix(model) == base_model
|
||||
assert has_context_1m_suffix(model) is (model != base_model)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model,expects_context_1m", [("claude-sonnet-5-5[1m]", True), ("claude-sonnet-5-5", False)])
|
||||
def test_validate_environment_adds_context_1m_beta_only_for_suffixed_model(
|
||||
model: str, expects_context_1m: bool
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
|
||||
headers: Final = AnthropicModelInfo().validate_environment(
|
||||
headers={"anthropic-beta": "interleaved-thinking-2025-05-14"},
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
api_key=FAKE_REGULAR_KEY,
|
||||
)
|
||||
|
||||
betas: Final = {beta.strip() for beta in headers["anthropic-beta"].split(",")}
|
||||
assert (ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_1M_2025_08_07.value in betas) is expects_context_1m
|
||||
assert "interleaved-thinking-2025-05-14" in betas
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue