mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge fa9cd8fed3 into 636e707089
This commit is contained in:
commit
6cb203dc21
8 changed files with 444 additions and 108 deletions
|
|
@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import (
|
|||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
|
||||
from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
|
||||
from litellm.llms.bedrock.common_utils import bedrock_model_is_unknown
|
||||
from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
|
@ -921,13 +922,15 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE: Final = (
|
|||
)
|
||||
|
||||
|
||||
def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPException) -> httpx.Response:
|
||||
def _bedrock_response_with_status(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response:
|
||||
original_response: Final = getattr(original_exception, "response", None)
|
||||
if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None:
|
||||
return httpx.Response(status_code=400, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"))
|
||||
if original_response.status_code == 400:
|
||||
return httpx.Response(
|
||||
status_code=status_code, request=httpx.Request(method="POST", url="https://api.openai.com/v1/")
|
||||
)
|
||||
if original_response.status_code == status_code:
|
||||
return original_response
|
||||
return httpx.Response(status_code=400, headers=original_response.headers, request=original_response.request)
|
||||
return httpx.Response(status_code=status_code, headers=original_response.headers, request=original_response.request)
|
||||
|
||||
|
||||
def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
|
||||
|
|
@ -963,7 +966,7 @@ def _map_bedrock_exception(
|
|||
message=mantle_context_window_message,
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
response=_bedrock_mantle_context_window_response(original_exception),
|
||||
response=_bedrock_response_with_status(original_exception, 400),
|
||||
)
|
||||
if (
|
||||
"too many tokens" in error_str
|
||||
|
|
@ -986,6 +989,13 @@ def _map_bedrock_exception(
|
|||
llm_provider="bedrock",
|
||||
response=getattr(original_exception, "response", None),
|
||||
)
|
||||
elif "The provided model identifier is invalid" in error_str and bedrock_model_is_unknown(model):
|
||||
raise NotFoundError(
|
||||
message=f"BedrockException - {error_str}",
|
||||
model=model,
|
||||
llm_provider="bedrock",
|
||||
response=_bedrock_response_with_status(original_exception, 404),
|
||||
)
|
||||
elif "Malformed input request" in error_str:
|
||||
raise BadRequestError(
|
||||
message=f"BedrockException - {error_str}",
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_image
|
|||
from litellm.llms.bedrock.common_utils import (
|
||||
bedrock_chat_uses_native_structured_output,
|
||||
bedrock_cleaned_output_schema,
|
||||
bedrock_model_is_unknown,
|
||||
bedrock_model_supports_regex_lookaround,
|
||||
bedrock_native_output_format,
|
||||
bedrock_native_output_is_opt_in,
|
||||
|
|
@ -529,13 +530,14 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"""
|
||||
Handle the reasoning_effort parameter based on the model type.
|
||||
|
||||
- GPT-OSS and DeepSeek V3 models: passed through unchanged via additionalModelRequestFields.
|
||||
- GPT-OSS, DeepSeek V3 and unknown models (``bedrock_model_is_unknown``): passed through unchanged
|
||||
via additionalModelRequestFields.
|
||||
- OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields.
|
||||
- Nova 2 models: transformed to reasoningConfig.
|
||||
- Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on
|
||||
adaptive Claude 4.6 / 4.7).
|
||||
"""
|
||||
if "gpt-oss" in model or "deepseek" in model:
|
||||
if "gpt-oss" in model or "deepseek" in model or bedrock_model_is_unknown(model):
|
||||
optional_params["reasoning_effort"] = reasoning_effort
|
||||
elif self._is_openai_gpt_reasoning_model(model):
|
||||
reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort}
|
||||
|
|
@ -681,7 +683,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"parallel_tool_calls",
|
||||
]
|
||||
|
||||
if "arn" in model: # we can't infer the model from the arn, so just add all params
|
||||
if "arn" in model or bedrock_model_is_unknown(model):
|
||||
supported_params.append("tools")
|
||||
supported_params.append("tool_choice")
|
||||
supported_params.append("thinking")
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import copy
|
|||
import json
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, cast, get_args
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
|
||||
|
|
@ -74,10 +74,6 @@ _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
|
|||
"{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
|
||||
)
|
||||
|
||||
_INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]).validate_python(
|
||||
get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL)
|
||||
)
|
||||
|
||||
|
||||
def _response_header(headers: Mapping[str, str], name: str) -> str | None:
|
||||
return headers.get(name, None)
|
||||
|
|
@ -562,62 +558,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
"""
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_bedrock_invoke_provider(
|
||||
model: str,
|
||||
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
|
||||
"""
|
||||
Helper function to get the bedrock provider from the model
|
||||
|
||||
handles 4 scenarios:
|
||||
1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
|
||||
2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic`
|
||||
3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama`
|
||||
4. model=us.amazon.nova-pro-v1:0 -> Returns `nova`
|
||||
"""
|
||||
if model.startswith("invoke/"):
|
||||
model = model.replace("invoke/", "", 1)
|
||||
|
||||
# Special case: Check for "nova" in model name first (before "amazon")
|
||||
# This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
|
||||
if "nova" in model.lower():
|
||||
if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
||||
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
|
||||
|
||||
_split_model: Final = model.split(".")[0]
|
||||
if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
||||
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
|
||||
|
||||
# If not a known provider, check for pattern with two slashes
|
||||
provider = AmazonInvokeConfig._get_provider_from_model_path(model)
|
||||
if provider is not None:
|
||||
return provider
|
||||
|
||||
for provider in _INVOKE_PROVIDERS:
|
||||
if provider in model:
|
||||
return provider
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_provider_from_model_path(
|
||||
model_path: str,
|
||||
) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None:
|
||||
"""
|
||||
Helper function to get the provider from a model path with format: provider/model-name
|
||||
|
||||
Args:
|
||||
model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name')
|
||||
|
||||
Returns:
|
||||
Optional[str]: The provider name, or None if no valid provider found
|
||||
"""
|
||||
parts: Final = model_path.split("/")
|
||||
if len(parts) >= 1:
|
||||
provider: Final = parts[0]
|
||||
if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
|
||||
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
|
||||
return None
|
||||
|
||||
def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]:
|
||||
# handle anthropic prompts and amazon titan prompts
|
||||
prompt = ""
|
||||
|
|
|
|||
|
|
@ -1098,6 +1098,11 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None:
|
|||
return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw)
|
||||
|
||||
|
||||
def _cost_map_row_names_a_provider(key: str) -> bool:
|
||||
entry: Final = _model_cost_entry(key)
|
||||
return entry is not None and entry.get("litellm_provider") is not None
|
||||
|
||||
|
||||
def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
|
||||
return tuple(
|
||||
_model_cost_entry(key)
|
||||
|
|
@ -1567,7 +1572,8 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
|
|||
|
||||
The deployment's ``model_info.supports_regex_lookaround`` wins, then the
|
||||
``model_prices_and_context_window.json`` entry of its ``base_model``, then the
|
||||
entry of ``model`` itself. A model nobody flagged keeps its schema as sent.
|
||||
entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless
|
||||
no candidate names a model LiteLLM knows, whatever route prefix it carries.
|
||||
"""
|
||||
params: Final = litellm_params or {}
|
||||
deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround")
|
||||
|
|
@ -1576,7 +1582,25 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
|
|||
base_model: Final = params.get("base_model")
|
||||
candidates: Final = (*((base_model,) if isinstance(base_model, str) else ()), model)
|
||||
flags: Final = (_bedrock_converse_model_flag(candidate, "supports_regex_lookaround") for candidate in candidates)
|
||||
return next((flag for flag in flags if flag is not None), True)
|
||||
flag: Final = next((flag for flag in flags if flag is not None), None)
|
||||
if flag is not None:
|
||||
return flag
|
||||
return not all(bedrock_model_is_unknown(split_bedrock_region_path(candidate)[1]) for candidate in candidates)
|
||||
|
||||
|
||||
def bedrock_model_is_unknown(model: str) -> bool:
|
||||
routed_id: Final = model.removeprefix("bedrock/")
|
||||
region, separator, regional_id = routed_id.partition("/")
|
||||
model_id: Final = regional_id if separator and region in get_all_bedrock_regions() else routed_id
|
||||
if "/" in model_id:
|
||||
return False
|
||||
base: Final = get_bedrock_base_model(model_id)
|
||||
regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions())
|
||||
if any(_cost_map_row_names_a_provider(key) for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)):
|
||||
return False
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
|
||||
return BaseAWSLLM.get_bedrock_invoke_provider(model_id) is None
|
||||
|
||||
|
||||
_BedrockConverseModelFlag: TypeAlias = Literal[
|
||||
|
|
@ -1585,10 +1609,13 @@ _BedrockConverseModelFlag: TypeAlias = Literal[
|
|||
]
|
||||
|
||||
|
||||
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
|
||||
def _bedrock_model_info_candidates(model: str) -> tuple[str, ...]:
|
||||
base: Final = get_bedrock_base_model(model)
|
||||
candidates: Final = dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base)))
|
||||
for candidate in candidates:
|
||||
return tuple(dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base))))
|
||||
|
||||
|
||||
def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None:
|
||||
for candidate in _bedrock_model_info_candidates(model):
|
||||
with contextlib.suppress(Exception):
|
||||
model_info = get_cached_model_info()(
|
||||
model=candidate,
|
||||
|
|
@ -1766,7 +1793,8 @@ class BedrockModelInfo(BaseLLMModelInfo):
|
|||
default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in;
|
||||
``request_params`` (the caller's chat params) sends such a request to Converse when it needs a
|
||||
feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family
|
||||
model stays on Converse without the prefix.
|
||||
model stays on Converse without the prefix. A model the cost map does not list as Converse goes to
|
||||
Invoke only when its id names an Invoke provider, and to Converse otherwise.
|
||||
"""
|
||||
route_mappings: dict[
|
||||
str,
|
||||
|
|
@ -1822,6 +1850,10 @@ class BedrockModelInfo(BaseLLMModelInfo):
|
|||
return "converse"
|
||||
if _OPENAI_FAMILY_MODEL_RE.search(base_model):
|
||||
return "converse"
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
|
||||
if BaseAWSLLM.get_bedrock_invoke_provider(model) is None:
|
||||
return "converse"
|
||||
return "invoke"
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -59,7 +59,6 @@ def test_openai_cyber_policy_uses_the_structured_error_code(shape: str, code: st
|
|||
assert caught.value.response.headers["x-request-id"] == "cyber-policy-request"
|
||||
|
||||
|
||||
|
||||
def _debug_info_for_api_base(configured_api_base: str) -> str:
|
||||
request: Final = httpx.Request("POST", "https://gateway.example/v1/chat/completions")
|
||||
original: Final = OpenAIError(
|
||||
|
|
@ -1601,6 +1600,24 @@ def test_bedrock_mantle_400_maps_to_bad_request():
|
|||
assert type(excinfo.value) is litellm.BadRequestError
|
||||
|
||||
|
||||
class _ErrorWithoutResponse(Exception):
|
||||
def __init__(self, message: str, status_code: int) -> None:
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
def test_bedrock_invalid_model_identifier_without_a_provider_response_still_answers_404():
|
||||
with pytest.raises(litellm.NotFoundError) as excinfo:
|
||||
exception_type(
|
||||
model="us.newvendor.typo-1",
|
||||
original_exception=_ErrorWithoutResponse("The provided model identifier is invalid.", 400),
|
||||
custom_llm_provider="bedrock",
|
||||
)
|
||||
|
||||
assert excinfo.value.response.status_code == 404
|
||||
|
||||
|
||||
def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded():
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
|
|
|
|||
|
|
@ -1221,6 +1221,97 @@ def test_transform_request_drops_lookaround_regex_when_the_deployment_model_info
|
|||
assert properties == _LOOKAROUND_FREE_PROPERTIES
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "litellm_params"),
|
||||
[
|
||||
("us.newvendor.model-1", None),
|
||||
("converse/us.newvendor.model-1", None),
|
||||
("us-east-1/us.newvendor.model-1", None),
|
||||
("us.newvendor.model-1", {"base_model": "bedrock/converse/us.newvendor.model-1"}),
|
||||
],
|
||||
)
|
||||
def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(model, litellm_params):
|
||||
assert "us.newvendor.model-1" not in litellm.model_cost
|
||||
|
||||
properties: Final = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL, litellm_params)
|
||||
|
||||
assert properties == _LOOKAROUND_FREE_PROPERTIES
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"arn:aws:bedrock:us-east-1:123456789012:custom-model-deployment/abc123",
|
||||
"arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123",
|
||||
"us.anthropic.claude-unreleased-v1:0",
|
||||
"eu.openai.gpt-unreleased",
|
||||
"apac.newvendor.listed-1",
|
||||
],
|
||||
)
|
||||
def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_litellm_still_recognizes(model, monkeypatch):
|
||||
monkeypatch.setitem(litellm.model_cost, "us.newvendor.listed-1", {"litellm_provider": "bedrock_converse"})
|
||||
|
||||
properties = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL)
|
||||
|
||||
assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"]
|
||||
|
||||
|
||||
def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_whose_deployment_opts_out():
|
||||
properties = _tool_schema_properties(
|
||||
"us.newvendor.model-1", _ARTIFACT_DATA_ANTHROPIC_TOOL, {"model_info": {"supports_regex_lookaround": True}}
|
||||
)
|
||||
|
||||
assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployment_params",
|
||||
[
|
||||
{"model": "bedrock/us.newvendor.model-1"},
|
||||
{"model": "bedrock/converse/us.newvendor.model-1", "allowed_openai_params": ["tools"]},
|
||||
],
|
||||
)
|
||||
def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaround_regex(
|
||||
respx_mock, deployment_params
|
||||
):
|
||||
route: Final = respx_mock.post(
|
||||
"https://bedrock-runtime.us-east-1.amazonaws.com/model/us.newvendor.model-1/converse"
|
||||
).mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2},
|
||||
},
|
||||
)
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "unlisted",
|
||||
"litellm_params": {
|
||||
**deployment_params,
|
||||
"aws_access_key_id": "fake-access-key",
|
||||
"aws_secret_access_key": "fake-secret-key",
|
||||
"aws_region_name": "us-east-1",
|
||||
},
|
||||
"model_info": {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
router.completion(
|
||||
model="unlisted",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[_ARTIFACT_DATA_OPENAI_TOOL],
|
||||
client=HTTPHandler(),
|
||||
)
|
||||
|
||||
tool: Final = json.loads(route.calls[0].request.content)["toolConfig"]["tools"][0]
|
||||
assert tool["toolSpec"]["inputSchema"]["json"]["properties"] == _LOOKAROUND_FREE_PROPERTIES
|
||||
|
||||
|
||||
def test_transform_request_keeps_lookaround_regex_when_the_deployment_model_info_opts_out():
|
||||
properties = _tool_schema_properties(
|
||||
"global.moonshotai.kimi-k3", _ARTIFACT_DATA_ANTHROPIC_TOOL, {"model_info": {"supports_regex_lookaround": True}}
|
||||
|
|
@ -9189,18 +9280,193 @@ def test_bedrock_tools_transformation_valid_params():
|
|||
assert "test" in result[0]["toolSpec"]["inputSchema"]["json"]["required"]
|
||||
|
||||
|
||||
def test_not_found_error():
|
||||
with pytest.raises(litellm.NotFoundError):
|
||||
completion(
|
||||
model="bedrock/bad_model",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is the meaning of life",
|
||||
}
|
||||
],
|
||||
def _invalid_model_identifier_reply(sent: list[httpx.Request]):
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
sent.append(request)
|
||||
return httpx.Response(
|
||||
400,
|
||||
json={"message": "The provided model identifier is invalid."},
|
||||
headers={"x-amzn-ErrorType": "ValidationException", "x-amzn-RequestId": "req-123"},
|
||||
)
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["bedrock/bad_model", "bedrock/us-east-1/bad_model"])
|
||||
def test_not_found_error(model):
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
with pytest.raises(litellm.NotFoundError) as exc_info:
|
||||
completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What is the meaning of life"}],
|
||||
client=HTTPHandler(
|
||||
client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent)))
|
||||
),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 404
|
||||
assert exc_info.value.response.status_code == 404
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-123"
|
||||
assert [request.url.raw_path for request in sent] == [b"/model/bad_model/converse"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, path",
|
||||
[
|
||||
("bedrock/converse/bad_model", b"/model/bad_model/converse"),
|
||||
("bedrock/anthropic.claude-typo-v1:0", b"/model/anthropic.claude-typo-v1:0/invoke"),
|
||||
],
|
||||
)
|
||||
def test_invalid_model_identifier_stays_a_bad_request_for_a_model_litellm_recognizes(model, path):
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
with pytest.raises(litellm.BadRequestError) as exc_info:
|
||||
completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What is the meaning of life"}],
|
||||
client=HTTPHandler(
|
||||
client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent)))
|
||||
),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert [request.url.raw_path for request in sent] == [path]
|
||||
|
||||
|
||||
_WEATHER_TOOL: Final = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather for a city",
|
||||
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _converse_tool_call_reply(sent: list[httpx.Request]):
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
sent.append(request)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"output": {
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}}
|
||||
],
|
||||
}
|
||||
},
|
||||
"stopReason": "tool_use",
|
||||
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2},
|
||||
},
|
||||
)
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["bedrock/us.newvendor.model-1", "bedrock/us-east-1/us.newvendor.model-1"])
|
||||
def test_model_missing_from_cost_map_sends_tools_to_converse(model):
|
||||
assert "us.newvendor.model-1" not in litellm.model_cost
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
response: Final = completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=[_WEATHER_TOOL],
|
||||
tool_choice="auto",
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert response.choices[0].message.tool_calls[0].function.name == "get_weather"
|
||||
assert [request.url.raw_path for request in sent] == [b"/model/us.newvendor.model-1/converse"]
|
||||
tool_config: Final = json.loads(sent[0].content)["toolConfig"]
|
||||
assert [tool["toolSpec"]["name"] for tool in tool_config["tools"]] == ["get_weather"]
|
||||
assert tool_config["toolChoice"] == {"auto": {}}
|
||||
|
||||
|
||||
def test_a_price_only_row_for_an_unlisted_model_still_sends_tools_to_converse(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost, "us.newvendor.priced-1", {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}
|
||||
)
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
completion(
|
||||
model="bedrock/us.newvendor.priced-1",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=[_WEATHER_TOOL],
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
tool_config: Final = json.loads(sent[0].content)["toolConfig"]
|
||||
assert [tool["toolSpec"]["name"] for tool in tool_config["tools"]] == ["get_weather"]
|
||||
|
||||
|
||||
def test_explicit_converse_route_to_an_unlisted_model_still_drops_tools_under_drop_params():
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
completion(
|
||||
model="bedrock/converse/us.newvendor.model-1",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=[_WEATHER_TOOL],
|
||||
drop_params=True,
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert "toolConfig" not in json.loads(sent[0].content)
|
||||
|
||||
|
||||
def test_unknown_model_gets_reasoning_effort_as_sent():
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
completion(
|
||||
model="bedrock/us.newvendor.model-1",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
reasoning_effort="high",
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert json.loads(sent[0].content)["additionalModelRequestFields"] == {"reasoning_effort": "high"}
|
||||
|
||||
|
||||
def test_mapped_model_without_function_calling_still_rejects_tools():
|
||||
model: Final = "bedrock/converse/meta.llama3-8b-instruct-v1:0"
|
||||
assert "meta.llama3-8b-instruct-v1:0" in litellm.model_cost
|
||||
assert not litellm.supports_function_calling(model=model)
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError):
|
||||
completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=[_WEATHER_TOOL],
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert sent == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, expected_base_model",
|
||||
|
|
|
|||
|
|
@ -214,24 +214,17 @@ def test_system_defined_inference_profile_arn_still_routes_to_converse():
|
|||
assert route == "converse"
|
||||
|
||||
|
||||
def test_other_opaque_arn_types_still_route_to_invoke():
|
||||
"""
|
||||
Only application-inference-profile ARNs are auto-routed to converse. Other
|
||||
opaque ARNs (provisioned-model, imported-model, custom-model-deployment)
|
||||
also yield no invoke provider, but they are frequently invoke-only with
|
||||
provider-specific body formats, so routing them to converse could break
|
||||
them. Guards the deliberate scope against an over-broad "any opaque ARN ->
|
||||
converse" generalization.
|
||||
"""
|
||||
for arn_segment in (
|
||||
"provisioned-model/abcdefgh1234",
|
||||
"imported-model/abcdefgh1234",
|
||||
"custom-model-deployment/abcdefgh1234",
|
||||
):
|
||||
route = BedrockModelInfo.get_bedrock_route(
|
||||
model=f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}"
|
||||
)
|
||||
assert route == "invoke", f"{arn_segment} should stay on invoke route"
|
||||
@pytest.mark.parametrize(
|
||||
"arn_segment",
|
||||
["provisioned-model/abcdefgh1234", "imported-model/abcdefgh1234", "custom-model-deployment/abcdefgh1234"],
|
||||
)
|
||||
def test_bare_opaque_arn_with_no_invoke_provider_routes_to_converse(arn_segment: str):
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
|
||||
model: Final = f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}"
|
||||
assert BaseAWSLLM.get_bedrock_invoke_provider(model) is None
|
||||
assert BedrockModelInfo.get_bedrock_route(model) == "converse"
|
||||
assert BedrockModelInfo.get_bedrock_route(f"bedrock/llama/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}") == "invoke"
|
||||
|
||||
|
||||
def test_govcloud_cross_region_inference_prefix():
|
||||
|
|
@ -1098,6 +1091,84 @@ def test_unmapped_openai_family_model_routes_to_converse():
|
|||
assert BedrockModelInfo.get_bedrock_route(imported) == "openai"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"unmapped",
|
||||
[
|
||||
"bedrock/minimax.minimax-m99-unmapped",
|
||||
"bedrock/us.nvidia.nemotron-99b-unmapped",
|
||||
"bedrock/writer.palmyra-x99-unmapped",
|
||||
"bedrock/zai.glm-99-unmapped",
|
||||
"bedrock/google.gemma-99-unmapped",
|
||||
"bedrock/global.xai.grok-99-unmapped",
|
||||
],
|
||||
)
|
||||
def test_unmapped_converse_only_family_model_routes_to_converse(unmapped: str):
|
||||
import litellm
|
||||
|
||||
assert BedrockModelInfo.get_base_model(unmapped) not in litellm.bedrock_converse_models
|
||||
assert BedrockModelInfo.get_bedrock_route(unmapped) == "converse"
|
||||
|
||||
|
||||
def test_unmapped_model_of_family_with_invoke_handler_still_routes_to_invoke():
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
|
||||
unmapped: Final = "bedrock/meta.llama99-unmapped-v1:0"
|
||||
assert BedrockModelInfo.get_base_model(unmapped) not in litellm.bedrock_converse_models
|
||||
assert BedrockModelInfo.get_bedrock_route(unmapped) == "invoke"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "model_path", "response_json"),
|
||||
[
|
||||
(
|
||||
"bedrock/us.newvendor.model-1",
|
||||
b"/model/us.newvendor.model-1/converse",
|
||||
{
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2},
|
||||
},
|
||||
),
|
||||
(
|
||||
"bedrock/mistral.mistral-99-unmapped-v1:0",
|
||||
b"/model/mistral.mistral-99-unmapped-v1:0/invoke",
|
||||
{"outputs": [{"text": "ok", "stop_reason": "stop"}]},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_completion_on_unmapped_model_posts_converse_unless_its_id_names_an_invoke_provider(
|
||||
model: str, model_path: bytes, response_json: dict[str, object]
|
||||
):
|
||||
import httpx
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
assert BedrockModelInfo.get_base_model(model) not in litellm.bedrock_converse_models
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
sent.append(request)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json=response_json,
|
||||
headers={"x-amzn-bedrock-input-token-count": "1", "x-amzn-bedrock-output-token-count": "1"},
|
||||
)
|
||||
|
||||
response: Final = litellm.completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))),
|
||||
aws_access_key_id="testing",
|
||||
aws_secret_access_key="testing",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "ok"
|
||||
assert [request.url.raw_path for request in sent] == [model_path]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "expected"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -24,10 +24,8 @@ class TestGetBedrockRoute:
|
|||
def test_nova2_prefix_routes_to_converse(self):
|
||||
assert BedrockModelInfo.get_bedrock_route(NOVA2_MODEL) == "converse"
|
||||
|
||||
def test_plain_arn_routes_to_invoke(self):
|
||||
# Without spec prefix, ARN doesn't match converse models
|
||||
result = BedrockModelInfo.get_bedrock_route(f"bedrock/{NOVA_ARN}")
|
||||
assert result == "invoke"
|
||||
def test_plain_arn_without_an_invoke_provider_routes_to_converse(self):
|
||||
assert BedrockModelInfo.get_bedrock_route(f"bedrock/{NOVA_ARN}") == "converse"
|
||||
|
||||
|
||||
class TestGetBedrockBaseModel:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue