mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-25 01:02:15 +00:00
fix(anthropic_messages): only Mantle consumes get_llm_provider's api_base
The /v1/messages handler passed the api_base get_llm_provider resolved to every provider's native messages config, which shadowed DEEPSEEK_ANTHROPIC_API_BASE and TENCENT_ANTHROPIC_API_BASE with the chat default and changed the azure_ai precedence. Messages configs now opt in through uses_get_llm_provider_api_base(), true only for Bedrock Mantle, whose region-prefixed model must resolve to a region host before the prefix is stripped. Also registers BedrockMantleAnthropicMessagesConfig in the lazy import registry.
This commit is contained in:
parent
368a839640
commit
3772993032
6 changed files with 61 additions and 1 deletions
|
|
@ -1684,6 +1684,9 @@ if TYPE_CHECKING:
|
|||
from .llms.bedrock.messages.mantle_transformation import (
|
||||
AmazonMantleMessagesConfig as AmazonMantleMessagesConfig,
|
||||
)
|
||||
from .llms.bedrock_mantle.messages.transformation import (
|
||||
BedrockMantleAnthropicMessagesConfig as BedrockMantleAnthropicMessagesConfig,
|
||||
)
|
||||
from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
|
||||
from .llms.together_ai.chat.transformation import (
|
||||
TogetherAIChatConfig as TogetherAIChatConfig,
|
||||
|
|
|
|||
|
|
@ -176,6 +176,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"BedrockClaudePlatformMessagesConfig",
|
||||
"AmazonAnthropicClaudeMessagesConfig",
|
||||
"AmazonMantleMessagesConfig",
|
||||
"BedrockMantleAnthropicMessagesConfig",
|
||||
"TogetherAIConfig",
|
||||
"TogetherAIChatConfig",
|
||||
"NLPCloudConfig",
|
||||
|
|
@ -746,6 +747,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
".llms.bedrock.messages.mantle_transformation",
|
||||
"AmazonMantleMessagesConfig",
|
||||
),
|
||||
"BedrockMantleAnthropicMessagesConfig": (
|
||||
".llms.bedrock_mantle.messages.transformation",
|
||||
"BedrockMantleAnthropicMessagesConfig",
|
||||
),
|
||||
"TogetherAIConfig": (".llms.together_ai.chat", "TogetherAIConfig"),
|
||||
"TogetherAIChatConfig": (
|
||||
".llms.together_ai.chat.transformation",
|
||||
|
|
|
|||
|
|
@ -501,7 +501,6 @@ def anthropic_messages_handler(
|
|||
api_base=litellm_params.api_base,
|
||||
api_key=litellm_params.api_key,
|
||||
)
|
||||
resolved_api_base: Final = dynamic_api_base if dynamic_api_base is not None else api_base
|
||||
|
||||
# Store agentic loop params in logging object for agentic hooks
|
||||
# This provides original request context needed for follow-up calls
|
||||
|
|
@ -652,6 +651,11 @@ def anthropic_messages_handler(
|
|||
"display": "summarized",
|
||||
}
|
||||
|
||||
resolved_api_base: Final = (
|
||||
dynamic_api_base
|
||||
if dynamic_api_base is not None and anthropic_messages_provider_config.uses_get_llm_provider_api_base()
|
||||
else api_base
|
||||
)
|
||||
return base_llm_http_handler.anthropic_messages_handler(
|
||||
model=model,
|
||||
messages=strip_provider_specific_fields_from_anthropic_messages(messages),
|
||||
|
|
|
|||
|
|
@ -128,6 +128,9 @@ class BaseAnthropicMessagesConfig(ABC):
|
|||
"""
|
||||
return True
|
||||
|
||||
def uses_get_llm_provider_api_base(self) -> bool:
|
||||
return False
|
||||
|
||||
def get_async_streaming_response_iterator(
|
||||
self,
|
||||
model: str,
|
||||
|
|
|
|||
|
|
@ -61,6 +61,9 @@ class BedrockMantleAnthropicMessagesConfig(BedrockMantleAuthMixin, AmazonMantleM
|
|||
def custom_llm_provider(self) -> str | None:
|
||||
return "bedrock_mantle"
|
||||
|
||||
def uses_get_llm_provider_api_base(self) -> bool:
|
||||
return True
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
|
|
|
|||
|
|
@ -1438,3 +1438,45 @@ async def test_anthropic_messages_leaves_non_provider_failures_unmapped():
|
|||
)
|
||||
|
||||
assert "Traceback" not in str(excinfo.value)
|
||||
|
||||
|
||||
def _recording_client(seen_urls: list[str]) -> AsyncHTTPHandler:
|
||||
def record_and_answer(request: httpx.Request) -> httpx.Response:
|
||||
seen_urls.append(str(request.url))
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_test",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "deepseek-chat",
|
||||
"content": [{"type": "text", "text": "pong"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 3, "output_tokens": 1},
|
||||
},
|
||||
)
|
||||
|
||||
upstream = AsyncHTTPHandler()
|
||||
upstream.client = httpx.AsyncClient(transport=httpx.MockTransport(record_and_answer))
|
||||
return upstream
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_provider_messages_api_base_env_is_not_shadowed_by_the_chat_default(monkeypatch):
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages import handler
|
||||
|
||||
monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False)
|
||||
monkeypatch.setenv("DEEPSEEK_ANTHROPIC_API_BASE", "https://deepseek.internal.example/anthropic")
|
||||
seen_urls: list[str] = []
|
||||
|
||||
await handler.anthropic_messages(
|
||||
max_tokens=16,
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
model="deepseek/deepseek-chat",
|
||||
api_key="sk-test",
|
||||
client=_recording_client(seen_urls),
|
||||
)
|
||||
|
||||
assert seen_urls == ["https://deepseek.internal.example/anthropic/v1/messages"]
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue