diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 651b169a9b2..f6340426c1b 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -191,7 +191,7 @@ class CustomStreamWrapper: custom_llm_provider: str | None = None, stream_options=None, make_call: Callable | None = None, - _response_headers: dict | None = None, + _response_headers: dict | httpx.Headers | None = None, ): self.model = model self.make_call = make_call diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 3a093e1f939..ca5f1298360 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -35,7 +35,7 @@ def make_sync_call( json_mode: bool | None = False, fake_stream: bool = False, stream_chunk_size: int | None = None, -): +) -> tuple[Any, httpx.Headers]: if client is None: client = _get_httpx_client() # Create a new client if none provided @@ -76,7 +76,7 @@ def make_sync_call( additional_args={"complete_input_dict": data}, ) - return completion_stream + return completion_stream, response.headers class BedrockConverseLLM(BaseAWSLLM): @@ -134,7 +134,7 @@ class BedrockConverseLLM(BaseAWSLLM): }, ) - completion_stream: Final = await make_call( + completion_stream, response_headers = await make_call( client=client, api_base=api_base, headers=dict(prepped.headers), @@ -151,6 +151,7 @@ class BedrockConverseLLM(BaseAWSLLM): model=model, custom_llm_provider="bedrock", logging_obj=logging_obj, + _response_headers=response_headers, ) return streaming_response @@ -232,7 +233,7 @@ class BedrockConverseLLM(BaseAWSLLM): except httpx.TimeoutException: raise BedrockError(status_code=408, message="Timeout error occurred.") - return litellm.AmazonConverseConfig()._transform_response( + transformed_response: Final = litellm.AmazonConverseConfig()._transform_response( model=model, response=response, model_response=model_response, @@ -244,6 +245,8 @@ class BedrockConverseLLM(BaseAWSLLM): optional_params=optional_params, encoding=encoding, ) + transformed_response.set_provider_response_headers(response.headers) + return transformed_response def completion( self, @@ -541,7 +544,7 @@ class BedrockConverseLLM(BaseAWSLLM): client = client if stream is not None and stream is True: - completion_stream: Final = make_sync_call( + completion_stream, response_headers = make_sync_call( client=(client if client is not None and isinstance(client, HTTPHandler) else None), api_base=proxy_endpoint_url, headers=prepped.headers, @@ -558,6 +561,7 @@ class BedrockConverseLLM(BaseAWSLLM): model=model, custom_llm_provider="bedrock", logging_obj=logging_obj, + _response_headers=response_headers, ) return streaming_response @@ -578,7 +582,7 @@ class BedrockConverseLLM(BaseAWSLLM): except httpx.TimeoutException: raise BedrockError(status_code=408, message="Timeout error occurred.") - return litellm.AmazonConverseConfig()._transform_response( + sync_transformed_response: Final = litellm.AmazonConverseConfig()._transform_response( model=model, response=response, model_response=model_response, @@ -590,3 +594,5 @@ class BedrockConverseLLM(BaseAWSLLM): optional_params=optional_params, encoding=encoding, ) + sync_transformed_response.set_provider_response_headers(response.headers) + return sync_transformed_response diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 2a125e38a82..ce89c6c23e2 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -163,7 +163,7 @@ async def make_call( json_mode: bool | None = False, bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None, stream_chunk_size: int | None = None, -): +) -> tuple[Any, httpx.Headers]: try: if client is None: client = get_async_httpx_client( @@ -225,7 +225,7 @@ async def make_call( additional_args={"complete_input_dict": data}, ) - return completion_stream + return completion_stream, response.headers except httpx.HTTPStatusError as err: error_code: Final = err.response.status_code raise BedrockError(status_code=error_code, message=err.response.text) @@ -248,7 +248,7 @@ def make_sync_call( json_mode: bool | None = False, bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None, stream_chunk_size: int | None = None, -): +) -> tuple[Any, httpx.Headers]: try: if client is None: client = _get_httpx_client( @@ -309,7 +309,7 @@ def make_sync_call( additional_args={"complete_input_dict": data}, ) - return completion_stream + return completion_stream, response.headers except httpx.HTTPStatusError as err: error_code: Final = err.response.status_code raise BedrockError(status_code=error_code, message=err.response.text) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 76f91aa9115..333326a766b 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -1,7 +1,6 @@ import copy import json import time -from functools import partial from typing import TYPE_CHECKING, Any, Final, cast, get_args import httpx @@ -446,24 +445,24 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): json_mode: bool | None = None, signed_json_body: bytes | None = None, ) -> CustomStreamWrapper: + completion_stream, response_headers = await make_call( + client=client, + api_base=api_base, + headers=headers, + data=json.dumps(data), + model=model, + messages=messages, + logging_obj=logging_obj, + fake_stream=True if "ai21" in api_base else False, + bedrock_invoke_provider=self.get_bedrock_invoke_provider(model), + json_mode=json_mode, + ) streaming_response: Final = CustomStreamWrapper( - completion_stream=None, - make_call=partial( - make_call, - client=client, - api_base=api_base, - headers=headers, - data=json.dumps(data), - model=model, - messages=messages, - logging_obj=logging_obj, - fake_stream=True if "ai21" in api_base else False, - bedrock_invoke_provider=self.get_bedrock_invoke_provider(model), - json_mode=json_mode, - ), + completion_stream=completion_stream, model=model, custom_llm_provider="bedrock", logging_obj=logging_obj, + _response_headers=response_headers, ) return streaming_response @@ -481,27 +480,28 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): json_mode: bool | None = None, signed_json_body: bytes | None = None, ) -> CustomStreamWrapper: - if client is None or isinstance(client, AsyncHTTPHandler): - client = _get_httpx_client(params={}) + sync_client: Final = ( + _get_httpx_client(params={}) if client is None or isinstance(client, AsyncHTTPHandler) else client + ) + completion_stream, response_headers = make_sync_call( + client=sync_client, + api_base=api_base, + headers=headers, + data=json.dumps(data), + signed_json_body=signed_json_body, + model=model, + messages=messages, + logging_obj=logging_obj, + fake_stream=True if "ai21" in api_base else False, + bedrock_invoke_provider=self.get_bedrock_invoke_provider(model), + json_mode=json_mode, + ) streaming_response: Final = CustomStreamWrapper( - completion_stream=None, - make_call=partial( - make_sync_call, - client=client, - api_base=api_base, - headers=headers, - data=json.dumps(data), - signed_json_body=signed_json_body, - model=model, - messages=messages, - logging_obj=logging_obj, - fake_stream=True if "ai21" in api_base else False, - bedrock_invoke_provider=self.get_bedrock_invoke_provider(model), - json_mode=json_mode, - ), + completion_stream=completion_stream, model=model, custom_llm_provider="bedrock", logging_obj=logging_obj, + _response_headers=response_headers, ) return streaming_response diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 9a950d7f920..8c98c526da1 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -635,6 +635,7 @@ class BaseLLMHTTPHandler: model=model, custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, + _response_headers=headers, ) if client is None or not isinstance(client, HTTPHandler): @@ -798,6 +799,7 @@ class BaseLLMHTTPHandler: model=model, custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, + _response_headers=_response_headers, ) return streamwrapper diff --git a/litellm/llms/sagemaker/chat/transformation.py b/litellm/llms/sagemaker/chat/transformation.py index 99543e7add1..37ddd813d6f 100644 --- a/litellm/llms/sagemaker/chat/transformation.py +++ b/litellm/llms/sagemaker/chat/transformation.py @@ -54,7 +54,30 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM): api_key: str | None = None, api_base: str | None = None, ) -> dict: - return headers + inference_component_name: Final = optional_params.get("model_id") + if not isinstance(inference_component_name, str): + return headers + return {**headers, "X-Amzn-SageMaker-Inference-Component": inference_component_name} + + def transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: matches the base chat transform signature + optional_params: dict, # mutable-ok: matches the base chat transform signature + litellm_params: dict, # mutable-ok: matches the base chat transform signature + headers: dict, # mutable-ok: matches the base chat transform signature + ) -> dict: # mutable-ok: the handler sends this body straight to httpx + request: Final = super().transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + served_model_name: Final = litellm_params.get("hf_model_name") + if not isinstance(served_model_name, str): + return request + return {**request, "model": served_model_name} def get_complete_url( self, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 79bc0b4b76c..91c10d13e8e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -49489,6 +49489,16 @@ "source": "https://docs.devin.ai/windsurf/plugins/cascade/models" }, "cognition/swe-1.7": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + "litellm_provider": "cognition", + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "source": "https://docs.devin.ai/desktop/models" + }, + "cognition/swe-1.7-lightning": { "input_cost_per_token": 2.5e-06, "output_cost_per_token": 1.25e-05, "cache_read_input_token_cost": 1e-06, @@ -49496,7 +49506,7 @@ "mode": "chat", "supports_function_calling": true, "supports_prompt_caching": true, - "source": "https://docs.devin.ai/windsurf/plugins/cascade/models" + "source": "https://docs.devin.ai/desktop/models" }, "pinstripes/ps/glm-4.5-air": { "max_tokens": 128000, diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 82e22bb5bbf..a8e545a8551 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -2640,51 +2640,63 @@ async def _process_team_members( return updated_users, updated_team_memberships +def _resolve_member_identity(member: Member, updated_users: Sequence[LiteLLM_UserTable]) -> Member: + """Return ``member`` with whichever of ``user_id`` / ``user_email`` the caller left out filled in. + + The roster entry is a snapshot, so whatever is missing here is missing for good. + Resolution runs both ways off the user rows the add just touched: added by email + -> stamp the user_id, added by user_id -> stamp the email. A value the caller + supplied is never overwritten. + """ + resolved_user_id: Final = member.user_id or next( + ( + user.user_id + for user in updated_users + if member.user_email is not None and user.user_email == member.user_email + ), + None, + ) + resolved_user_email: Final = member.user_email or next( + ( + user.user_email + for user in updated_users + if resolved_user_id is not None and user.user_id == resolved_user_id and user.user_email is not None + ), + None, + ) + return member.model_copy( + update={ # mutable-ok: pydantic update payload + "user_id": resolved_user_id, + "user_email": resolved_user_email, + } + ) + + +def _member_already_in_team(member: Member, complete_team_data: LiteLLM_TeamTable) -> bool: + return any( + (member.user_id is not None and existing_member.user_id == member.user_id) + or (member.user_email is not None and existing_member.user_email == member.user_email) + for existing_member in complete_team_data.members_with_roles + ) + + async def _update_team_members_list( data: TeamMemberAddRequest, complete_team_data: LiteLLM_TeamTable, updated_users: list[LiteLLM_UserTable], ) -> None: """Update the team's members_with_roles list.""" - if isinstance(data.member, Member): - new_member: Final = data.member.model_copy() + requested_members: Final[Sequence[Member]] = ( + (data.member,) if isinstance(data.member, Member) else tuple(data.member) + ) + resolved_members: Final = tuple(_resolve_member_identity(m, updated_users) for m in requested_members) - # get user id - if new_member.user_id is None and new_member.user_email is not None: - for user in updated_users: - if user.user_email is not None and user.user_email == new_member.user_email: - new_member.user_id = user.user_id - - # Check if member already exists in team before adding - member_already_exists = False - for existing_member in complete_team_data.members_with_roles: - if (new_member.user_id is not None and existing_member.user_id == new_member.user_id) or ( - new_member.user_email is not None and existing_member.user_email == new_member.user_email - ): - member_already_exists = True - break - - if not member_already_exists: - complete_team_data.members_with_roles.append(new_member) - - elif isinstance(data.member, list): - for nm in data.member: - if nm.user_id is None and nm.user_email is not None: - for user in updated_users: - if user.user_email is not None and user.user_email == nm.user_email: - nm.user_id = user.user_id - - # Check if member already exists in team before adding - member_already_exists = False - for existing_member in complete_team_data.members_with_roles: - if (nm.user_id is not None and existing_member.user_id == nm.user_id) or ( - nm.user_email is not None and existing_member.user_email == nm.user_email - ): - member_already_exists = True - break - - if not member_already_exists: - complete_team_data.members_with_roles.append(nm) + # extend() consumes the generator as it appends, so a member already added by this + # same call is seen by the next _member_already_in_team check - the batch dedupes + # against itself exactly as the append-one-at-a-time loop this replaced did. + complete_team_data.members_with_roles.extend( # rebind-ok: this helper's contract is to grow the caller's roster in place + m for m in resolved_members if not _member_already_in_team(m, complete_team_data) + ) async def _add_team_members_to_team( @@ -4086,6 +4098,39 @@ async def _add_team_member_budget_table( return team_info_response_object +async def _hydrate_member_emails( + prisma_client: PrismaClient, + members: Sequence[Member], +) -> tuple[Member, ...]: + """Fill in ``user_email`` for roster entries that were stored without one. + + ``members_with_roles`` is a denormalized snapshot written at add-time, so an entry + stored with ``user_email=None`` keeps that null even once the user row has an email. + Look the missing ones up in ``LiteLLM_UserTable`` (one indexed query) and fill them + in. A stored email is never overwritten - the snapshot stays the source of truth + wherever it has a value. + """ + missing_user_ids: Final = frozenset(m.user_id for m in members if not m.user_email and m.user_id is not None) + if not missing_user_ids: + return tuple(members) + + user_rows: Final[Sequence[LiteLLM_UserTable]] = await _user_db(prisma_client).find_many( + where={ # mutable-ok: Prisma query filters are dict-shaped + "user_id": { # mutable-ok: Prisma query filters are dict-shaped + "in": sorted(missing_user_ids) + } + } + ) + email_by_user_id: Final = MappingProxyType({u.user_id: u.user_email for u in user_rows if u.user_email}) + + return tuple( + m.model_copy(update={"user_email": email_by_user_id[m.user_id]}) # mutable-ok: pydantic update payload + if not m.user_email and m.user_id in email_by_user_id + else m + for m in members + ) + + async def _resolve_team_access_group_resources( _team_info: TeamInfoResponseObjectTeamTable, ) -> TeamInfoResponseObjectTeamTable: @@ -4221,9 +4266,22 @@ async def team_info( # Resolve resources inherited from access groups resolved_team_info: Final = await _resolve_team_access_group_resources(_team_info) + # Fill in emails the add-time roster snapshot never captured + hydrated_members: Final = await _hydrate_member_emails( + prisma_client=prisma_client, + members=resolved_team_info.members_with_roles, + ) + hydrated_team_info: Final = resolved_team_info.model_copy( + update={ # mutable-ok: pydantic update payload + # list(), not the tuple: model_copy skips validation, so the field has + # to be handed the list[Member] the response model declares. + "members_with_roles": list(hydrated_members) # mutable-ok: declared list[Member] + } + ) + response_object: Final = TeamInfoResponseObject( team_id=team_id, - team_info=resolved_team_info, + team_info=hydrated_team_info, keys=keys, team_memberships=returned_tm, ) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 822f03873b3..a9f1dc990db 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -10,6 +10,7 @@ from pydantic import BaseModel import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import ( + LITELLM_PROXY_MASTER_KEY_ALIAS, LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, REDACTED_BY_LITELM_STRING, @@ -21,6 +22,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, reconstruct_model_name, ) +from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error @@ -53,13 +55,6 @@ def _get_max_string_length_prompt_in_db() -> int: return DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB -def _hash_api_key_for_spend_log(api_key: str) -> str: - stripped: Final = api_key[7:] if api_key[:7].lower() == "bearer " else api_key - if stripped.startswith("sk-"): - return hash_token(stripped) - return stripped - - def _is_master_key(api_key: str | None, _master_key: str | None) -> bool: """ Raw-only constant-time master-key comparison. The hashed form is never @@ -70,6 +65,28 @@ def _is_master_key(api_key: str | None, _master_key: str | None) -> bool: return secrets.compare_digest(api_key, _master_key) +_HASHED_JWT_RE = re.compile(r"hashed-jwt-[a-fA-F0-9]{64}") + + +def _is_non_secret_key_value(value: str) -> bool: + return ( + value == LITELLM_PROXY_MASTER_KEY_ALIAS + or is_valid_sha256_hash(value) + or _HASHED_JWT_RE.fullmatch(value) is not None + ) + + +def _redact_logged_api_key(value: str | None, *, already_redacted: bool = False) -> str | None: + if not isinstance(value, str) or not value: + return None + stripped: Final = re.sub(r"(?i)^bearer ", "", value) + if not stripped: + return None + if already_redacted and _is_non_secret_key_value(stripped): + return stripped + return hash_token(stripped) + + def _get_spend_logs_metadata( metadata: dict | None, applied_guardrails: list[str] | None = None, @@ -123,9 +140,12 @@ def _get_spend_logs_metadata( # Filter the metadata dictionary to include only the specified keys clean_metadata: Final = SpendLogsMetadata(**{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__}) - raw_user_api_key: Final = clean_metadata.get("user_api_key") - if raw_user_api_key is not None and isinstance(raw_user_api_key, str): - clean_metadata["user_api_key"] = _hash_api_key_for_spend_log(raw_user_api_key) + _raw_key: Final = clean_metadata.get("user_api_key") + _trusted_hash: Final = metadata.get("user_api_key_hash") + _already_redacted: Final = ( + isinstance(_trusted_hash, str) and _is_non_secret_key_value(_trusted_hash) and _trusted_hash == _raw_key + ) + clean_metadata["user_api_key"] = _redact_logged_api_key(_raw_key, already_redacted=_already_redacted) clean_metadata["applied_guardrails"] = applied_guardrails clean_metadata["batch_models"] = batch_models clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata @@ -281,16 +301,23 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs standard_logging_prompt_tokens = standard_logging_payload.get("prompt_tokens", 0) standard_logging_completion_tokens = standard_logging_payload.get("completion_tokens", 0) standard_logging_total_tokens = standard_logging_payload.get("total_tokens", 0) - if api_key is not None and isinstance(api_key, str): - api_key = _hash_api_key_for_spend_log(api_key) + _trusted_hash = metadata.get("user_api_key_hash") + _key_already_redacted = ( + isinstance(_trusted_hash, str) and _is_non_secret_key_value(_trusted_hash) and _trusted_hash == api_key + ) + api_key = _redact_logged_api_key(api_key, already_redacted=_key_already_redacted) or "" if ( standard_logging_payload is not None ): # [TODO] migrate completely to sl payload. currently missing pass-through endpoint data - api_key = api_key or standard_logging_payload["metadata"].get("user_api_key_hash") or "" + api_key = ( + api_key + or _redact_logged_api_key( + standard_logging_payload["metadata"].get("user_api_key_hash"), already_redacted=True + ) + or "" + ) end_user_id = end_user_id or standard_logging_payload["metadata"].get("user_api_key_end_user_id") - # BUG FIX: Don't overwrite api_key when standard_logging_payload is None - # The api_key was already extracted from metadata (line 243) and hashed (lines 256-259) request_tags = safe_dumps(metadata.get("tags", [])) if isinstance(metadata.get("tags", []), list) else "[]" if ( standard_logging_payload is not None and standard_logging_payload.get("request_tags") is not None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index e6138101899..d3effbdfd34 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -11,6 +11,7 @@ from typing import ( get_args, ) +import httpx from openai._models import BaseModel as OpenAIObject from openai.types.audio.transcription_create_params import ( FileTypes as FileTypes, @@ -49,7 +50,7 @@ from litellm.types.llms.base import ( ) from litellm.types.mcp import MCPServerCostInfo -from ..litellm_core_utils.core_helpers import map_finish_reason +from ..litellm_core_utils.core_helpers import map_finish_reason, process_response_headers from .agents import LiteLLMSendMessageResponse from .guardrails import GuardrailEventHooks from .llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse @@ -1916,6 +1917,10 @@ class ModelResponseBase(OpenAIObject): _response_headers: dict | None = None + def set_provider_response_headers(self, headers: httpx.Headers) -> None: + """Surface a provider's raw response headers to the caller as `llm_provider-*` headers.""" + self._hidden_params["additional_headers"] = process_response_headers(headers) + def model_dump(self, **kwargs): """Default to exclude_unset to avoid Pydantic serializer warnings for OpenAIObject-derived types.""" if "exclude_unset" not in kwargs and "exclude_none" not in kwargs: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 79bc0b4b76c..91c10d13e8e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -49489,6 +49489,16 @@ "source": "https://docs.devin.ai/windsurf/plugins/cascade/models" }, "cognition/swe-1.7": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + "litellm_provider": "cognition", + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "source": "https://docs.devin.ai/desktop/models" + }, + "cognition/swe-1.7-lightning": { "input_cost_per_token": 2.5e-06, "output_cost_per_token": 1.25e-05, "cache_read_input_token_cost": 1e-06, @@ -49496,7 +49506,7 @@ "mode": "chat", "supports_function_calling": true, "supports_prompt_caching": true, - "source": "https://docs.devin.ai/windsurf/plugins/cascade/models" + "source": "https://docs.devin.ai/desktop/models" }, "pinstripes/ps/glm-4.5-air": { "max_tokens": 128000, diff --git a/ruff-tests.toml b/ruff-tests.toml index 6e77f4792a7..60438d355f0 100644 --- a/ruff-tests.toml +++ b/ruff-tests.toml @@ -20,6 +20,12 @@ # PT012 a `pytest.raises` block that runs on past the raising call. Everything after # that call is dead, so an `assert` sitting there is never checked. Keep the # block to the call itself and put the assertions below it +# PT011 `pytest.raises(Exception)` / `(ValueError)` / `(OSError)` with no `match=`. The +# block passes on any error that broad, so the TypeError a refactor introduced +# reads as the rejection under test. Pin the message the code actually raises +# PT014 the same `parametrize` case listed twice. The copy re-runs an assertion that +# already passed and adds no coverage, and it usually marks a case someone meant +# to vary and forgot to edit # # No target-version here on purpose: it resolves from requires-python (>=3.10), so # 3.11-only builtins like BaseExceptionGroup are correctly flagged in a tree that @@ -27,4 +33,4 @@ line-length = 120 -lint.select = ["F821", "B011", "B015", "B017", "B018", "PT012", "PT015", "PLR0133", "PLW0127"] +lint.select = ["F821", "B011", "B015", "B017", "B018", "PT011", "PT012", "PT014", "PT015", "PLR0133", "PLW0127"] diff --git a/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py b/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py index 0cd6055e09d..bdf73b6ab03 100644 --- a/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py +++ b/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py @@ -400,7 +400,7 @@ def test_invalid_metric_name_validation(): litellm.prometheus_metrics_config = test_config # Creating PrometheusLogger should raise ValueError due to invalid metric - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Configuration validation failed') as exc_info: PrometheusLogger() # Verify error message contains information about invalid metric @@ -429,7 +429,7 @@ def test_invalid_labels_validation(): litellm.prometheus_metrics_config = test_config # Creating PrometheusLogger should raise ValueError due to invalid labels - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Configuration validation failed') as exc_info: PrometheusLogger() # Verify error message contains information about invalid labels @@ -598,7 +598,7 @@ def test_invalid_exclude_metric_name_raises(reset_prometheus_exclude_settings): litellm.prometheus_exclude_labels = None litellm.prometheus_exclude_metrics = ["not_a_real_metric"] - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Prometheus exclude configuration validation failed') as exc_info: PrometheusLogger() assert "not_a_real_metric" in str(exc_info.value) @@ -612,7 +612,7 @@ def test_invalid_exclude_label_name_raises(reset_prometheus_exclude_settings): litellm.prometheus_exclude_metrics = None litellm.prometheus_exclude_labels = ["not_a_real_label"] - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Prometheus exclude configuration validation failed') as exc_info: PrometheusLogger() assert "not_a_real_label" in str(exc_info.value) diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py index f257b47404e..6b6b5d768dd 100644 --- a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py +++ b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py @@ -141,7 +141,7 @@ async def test_bedrock_apply_guardrail_api_failure(): mock_api_request.side_effect = Exception("API connection failed") # Test the apply_guardrail method should raise an exception - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Bedrock guardrail failed: API connection failed') as exc_info: await guardrail.apply_guardrail( inputs={"texts": ["This is a test message"]}, request_data={}, diff --git a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py index 714f3be6df9..2d845a445b5 100644 --- a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py +++ b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py @@ -1653,7 +1653,7 @@ async def test_afile_retrieve_raises_error_when_no_router_and_file_object_none() unified_file_id = "test-unified-file-id" - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='LiteLLM Managed File object with id=test-unified-file-id') as exc_info: await proxy_managed_files.afile_retrieve( file_id=unified_file_id, litellm_parent_otel_span=None, @@ -1719,7 +1719,7 @@ async def test_afile_retrieve_raises_error_for_non_managed_file(): # Mock get_unified_file_id to return None (file not found) proxy_managed_files.get_unified_file_id = AsyncMock(return_value=None) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='LiteLLM Managed File object with id=non-existent-file-id') as exc_info: await proxy_managed_files.afile_retrieve( file_id="non-existent-file-id", litellm_parent_otel_span=None, @@ -2027,7 +2027,7 @@ async def test_list_batches_from_managed_objects_table_provider_filter_raises_ex ) # Filtering by provider should raise Exception - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Filtering by 'provider' is not supported when using managed") as exc_info: await proxy_managed_files.list_user_batches( user_api_key_dict=UserAPIKeyAuth(user_id="test-user"), limit=10, @@ -2053,7 +2053,7 @@ async def test_list_batches_from_managed_objects_table_target_model_name_filter_ ) # Filtering by provider should raise Exception - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Filtering by 'target_model_names' is not supported when") as exc_info: await proxy_managed_files.list_user_batches( user_api_key_dict=UserAPIKeyAuth(user_id="test-user"), limit=10, diff --git a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py index bd6637ffcac..ed6735a7126 100644 --- a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py +++ b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py @@ -448,7 +448,7 @@ def test_check_team_project_limits_models_not_in_team(): models=["gpt-5.5", "claude-3"], # claude-3 not in team ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="not in team's allowed models\\. Team allowed models") as exc_info: _check_team_project_limits(team_object=team, data=data) assert "claude-3" in str(exc_info.value.detail) @@ -476,7 +476,7 @@ def test_check_team_project_limits_budget_exceeds_team(): max_budget=150.0, # exceeds team's 100.0 ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Project max_budget') as exc_info: _check_team_project_limits(team_object=team, data=data) assert "exceeds team's max_budget" in str(exc_info.value.detail) @@ -551,7 +551,7 @@ def test_check_team_project_limits_tpm_exceeds_team(): tpm_limit=20000, # exceeds team's 10000 ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Project tpm_limit') as exc_info: _check_team_project_limits(team_object=team, data=data) assert "exceeds team's tpm_limit" in str(exc_info.value.detail) @@ -577,7 +577,7 @@ def test_check_team_project_limits_negative_budget(): max_budget=-10.0, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='max_budget cannot be negative\\. Received') as exc_info: _check_team_project_limits(team_object=team, data=data) assert "cannot be negative" in str(exc_info.value.detail) @@ -604,7 +604,7 @@ def test_check_team_project_limits_soft_budget_gte_max(): soft_budget=100.0, # equal to max, should fail ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='must be strictly lower than max_budget') as exc_info: _check_team_project_limits(team_object=team, data=data) assert "must be strictly lower" in str(exc_info.value.detail) diff --git a/tests/guardrails_tests/test_dynamoai_guardrails.py b/tests/guardrails_tests/test_dynamoai_guardrails.py index 98f676a71d5..6f0ea00165b 100644 --- a/tests/guardrails_tests/test_dynamoai_guardrails.py +++ b/tests/guardrails_tests/test_dynamoai_guardrails.py @@ -61,7 +61,7 @@ async def test_dynamoai_blocks_content_with_block_action(): guardrail.should_run_guardrail = MagicMock(return_value=True) # Test that the guardrail raises ValueError for blocked content - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='violation\\(s\\) detected') as exc_info: await guardrail.async_pre_call_hook( data=request_data, user_api_key_dict=UserAPIKeyAuth(), diff --git a/tests/guardrails_tests/test_eu_ai_act_article5.py b/tests/guardrails_tests/test_eu_ai_act_article5.py index 0903e6c5416..f7384667481 100644 --- a/tests/guardrails_tests/test_eu_ai_act_article5.py +++ b/tests/guardrails_tests/test_eu_ai_act_article5.py @@ -211,7 +211,7 @@ class TestEUAIActArticle5ConditionalMatching: # Apply guardrail if expected == "BLOCK": # Should raise an exception or return modified response indicating block - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Content blocked: eu_ai_act_article') as exc_info: await content_filter_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, diff --git a/tests/guardrails_tests/test_eu_ai_act_french_3_scenarios.py b/tests/guardrails_tests/test_eu_ai_act_french_3_scenarios.py index 6b9774d9cde..221ca5aa6e6 100644 --- a/tests/guardrails_tests/test_eu_ai_act_french_3_scenarios.py +++ b/tests/guardrails_tests/test_eu_ai_act_french_3_scenarios.py @@ -83,7 +83,7 @@ class TestEUAIActFrench3Scenarios: print(f"{'='*70}\n") # Should raise an exception (blocked) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="prohibited_practices_fr conditional match 'concevoir \\+") as exc_info: await content_filter_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, @@ -123,7 +123,7 @@ class TestEUAIActFrench3Scenarios: print(f"{'='*70}\n") # Should raise an exception (blocked) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="prohibited_practices_fr conditional match 'créer \\+") as exc_info: await content_filter_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, @@ -194,7 +194,7 @@ class TestEUAIActFrench3Scenarios: print(f"{'='*70}\n") # Should raise an exception (blocked by conditional matching) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="prohibited_practices_fr conditional match 'développer \\+") as exc_info: await content_filter_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, @@ -278,7 +278,7 @@ class TestFrenchEdgeCases: request_data = {"messages": [{"role": "user", "content": sentence}]} # Should still block (no exception bypass) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="prohibited_practices_fr conditional match 'créer \\+ crédit") as exc_info: await content_filter_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, diff --git a/tests/guardrails_tests/test_sg_mas_ai_guardrails.py b/tests/guardrails_tests/test_sg_mas_ai_guardrails.py index 668ee704692..e587d666a79 100644 --- a/tests/guardrails_tests/test_sg_mas_ai_guardrails.py +++ b/tests/guardrails_tests/test_sg_mas_ai_guardrails.py @@ -55,7 +55,7 @@ def _make_guardrail(yaml_filename: str, category_name: str) -> ContentFilterGuar async def _expect_block(guardrail: ContentFilterGuardrail, sentence: str, reason: str): request_data = {"messages": [{"role": "user", "content": sentence}]} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Content blocked: sg_mas_') as exc_info: await guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, diff --git a/tests/guardrails_tests/test_sg_pdpa_guardrails.py b/tests/guardrails_tests/test_sg_pdpa_guardrails.py index fd7133bc745..42c3a15f9f6 100644 --- a/tests/guardrails_tests/test_sg_pdpa_guardrails.py +++ b/tests/guardrails_tests/test_sg_pdpa_guardrails.py @@ -62,7 +62,7 @@ def _make_guardrail(yaml_filename: str, category_name: str) -> ContentFilterGuar async def _expect_block(guardrail: ContentFilterGuardrail, sentence: str, reason: str): """Assert that the guardrail BLOCKS the sentence.""" request_data = {"messages": [{"role": "user", "content": sentence}]} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Content blocked: sg_pdpa_') as exc_info: await guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, diff --git a/tests/litellm_utils_tests/test_hashicorp.py b/tests/litellm_utils_tests/test_hashicorp.py index 9aff7ddc10e..fa39a045227 100644 --- a/tests/litellm_utils_tests/test_hashicorp.py +++ b/tests/litellm_utils_tests/test_hashicorp.py @@ -432,7 +432,7 @@ def test_hashicorp_get_url_rejects_path_traversal(monkeypatch, malicious_secret_ monkeypatch.setenv("HCP_VAULT_TOKEN", "test-token-for-get-url-only") manager = HashicorpSecretManager() - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Invalid secret_name'): manager.get_url(malicious_secret_name) diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py index 2b539b97c9b..3c73224d7a1 100644 --- a/tests/litellm_utils_tests/test_utils.py +++ b/tests/litellm_utils_tests/test_utils.py @@ -2147,7 +2147,7 @@ def test_validate_user_messages_invalid_content_type(): messages = [{"content": [{"type": "invalid_type", "text": "Hello"}]}] - with pytest.raises(Exception) as e: + with pytest.raises(Exception, match='Please ensure all messages are valid OpenAI chat completion') as e: validate_chat_completion_user_messages(messages) assert "Invalid message" in str(e) diff --git a/tests/litellm_utils_tests/test_validate_tool_choice.py b/tests/litellm_utils_tests/test_validate_tool_choice.py index 0e6294a7cd4..07f8c9ed8f4 100644 --- a/tests/litellm_utils_tests/test_validate_tool_choice.py +++ b/tests/litellm_utils_tests/test_validate_tool_choice.py @@ -37,27 +37,27 @@ def test_validate_tool_choice_cursor_format(): def test_validate_tool_choice_invalid_dict(): """Test that invalid dict formats raise exceptions.""" # Missing both type and function - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Invalid tool choice, tool_choice=\\{\\}\\. Please ensure') as exc_info: validate_chat_completion_tool_choice({}) assert "Invalid tool choice" in str(exc_info.value) # Invalid type value - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Invalid tool choice, tool_choice=\\{'type': 'invalid'\\}\\.") as exc_info: validate_chat_completion_tool_choice({"type": "invalid"}) assert "Invalid tool choice" in str(exc_info.value) # Has type but missing function when type is "function" - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Invalid tool choice, tool_choice=\\{'type': 'function'\\}\\.") as exc_info: validate_chat_completion_tool_choice({"type": "function"}) assert "Invalid tool choice" in str(exc_info.value) def test_validate_tool_choice_invalid_type(): """Test that invalid types raise exceptions.""" - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="\\. Expecting str, or dict\\. Please ensure") as exc_info: validate_chat_completion_tool_choice(123) assert "Got=" in str(exc_info.value) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Invalid tool choice, tool_choice=\\[\\]\\. Got=\\.") as exc_info: validate_chat_completion_tool_choice([]) assert "Got=" in str(exc_info.value) diff --git a/tests/llm_responses_api_testing/test_responses_hooks.py b/tests/llm_responses_api_testing/test_responses_hooks.py index 2344a62de4d..66dbb29dba5 100644 --- a/tests/llm_responses_api_testing/test_responses_hooks.py +++ b/tests/llm_responses_api_testing/test_responses_hooks.py @@ -295,7 +295,7 @@ async def test_responses_streaming_failure_triggers_failure_handlers(): call_type=CallTypes.responses.value, ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="boom"): iterator._process_chunk('{"delta": "chunk"}') # allow failure callbacks to run diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 3303fafafb0..9534bc8de3c 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -1890,7 +1890,7 @@ def test_bedrock_completion_test_4(modify_params): ] assert transformed_messages == expected_messages else: - with pytest.raises(Exception) as e: + with pytest.raises(Exception, match=r"litellm\.modify_params") as e: litellm.completion(**data) assert "litellm.modify_params" in str(e.value) diff --git a/tests/llm_translation/test_bedrock_moonshot.py b/tests/llm_translation/test_bedrock_moonshot.py index a9f4a86b3b6..a82d1c6f029 100644 --- a/tests/llm_translation/test_bedrock_moonshot.py +++ b/tests/llm_translation/test_bedrock_moonshot.py @@ -12,6 +12,7 @@ This test suite verifies: """ from base_llm_unit_tests import BaseLLMChatTest +import httpx import pytest import sys import os @@ -208,14 +209,6 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest): endpoint with the messages body. Iteration of the stream itself is not exercised here — moonshot streaming delegates to the OpenAI parser and is covered by the OpenAI test suite. - - Note: bedrock invoke streaming cannot be intercepted by patching - the caller-supplied client, because ``CustomStreamWrapper.fetch_sync_stream`` - at streaming_handler.py invokes the stored ``make_call`` partial with - ``client=litellm.module_level_client``, which overrides any client the - caller passed. Patch ``make_sync_call`` at its import site in - ``base_invoke_transformation`` so we observe the exact kwargs the - partial was built with at stream-wrapper construction time. """ from litellm.utils import CustomStreamWrapper @@ -225,7 +218,7 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest): captured.update(kwargs) # Return an empty iterator so the stream wrapper's iteration # doesn't try to parse real bytes. - return iter([]) + return iter([]), httpx.Headers() with patch( "litellm.llms.bedrock.chat.invoke_transformations." @@ -246,11 +239,6 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest): aws_region_name="us-west-2", ) assert isinstance(response, CustomStreamWrapper) - # Trigger fetch_sync_stream → make_call(...) → fake_make_sync_call. - try: - next(iter(response)) - except StopIteration: - pass assert captured, "make_sync_call was never invoked" assert captured["api_base"].endswith("/invoke-with-response-stream") diff --git a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py index cc493cc5a28..8c7390d3d04 100644 --- a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py +++ b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py @@ -982,7 +982,7 @@ def test_convert_to_model_response_object_with_real_error(): }, } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception) as exc_info: # noqa: PT011 # message rides on .message, str() is empty convert_to_model_response_object( model_response_object=ModelResponse(), response_object=response_object, @@ -1243,7 +1243,7 @@ def test_convert_to_model_response_object_with_error_code_only(): }, } - with pytest.raises(Exception) as exc_info: # noqa: B017 # bare Exception raised, so status_code is the assertion + with pytest.raises(Exception) as exc_info: # noqa: B017, PT011 # bare Exception, empty message, so status_code is the assertion convert_to_model_response_object( model_response_object=ModelResponse(), response_object=response_object, @@ -1423,7 +1423,7 @@ def test_error_message_includes_function_args(): "choices": [{"index": 0}], } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='in convert_to_model_response_object') as exc_info: convert_to_model_response_object( model_response_object=ModelResponse(), response_object=response_object, diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index c3519fcb40f..1b4c8a82cf4 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -1845,7 +1845,7 @@ def test_parse_tool_call_arguments_malformed_json(): parse_tool_call_arguments, ) - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="Failed to parse tool call arguments for tool 'load_skill") as exc_info: parse_tool_call_arguments( '{"skill_name": "pptx', tool_name="load_skill", @@ -1877,7 +1877,7 @@ def test_convert_to_anthropic_tool_invoke_malformed_json(): } ] - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="Failed to parse tool call arguments for tool 'bad_tool") as exc_info: convert_to_anthropic_tool_invoke(tool_calls) error_msg = str(exc_info.value) @@ -2023,7 +2023,7 @@ def test_parse_tool_call_arguments_still_raises_for_unrepairable(): parse_tool_call_arguments, ) - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="Failed to parse tool call arguments for tool 'test_tool") as exc_info: parse_tool_call_arguments( '{"key": "unterminated', tool_name="test_tool", diff --git a/tests/llm_translation/test_triton.py b/tests/llm_translation/test_triton.py index 21887e8d848..f4a26360a6c 100644 --- a/tests/llm_translation/test_triton.py +++ b/tests/llm_translation/test_triton.py @@ -45,7 +45,7 @@ def test_split_embedding_by_shape_fails_with_shape_value_error(): "data": [1, 2, 3, 4, 5, 6], } ] - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Shape must be of length'): TritonEmbeddingConfig.split_embedding_by_shape( data[0]["data"], data[0]["shape"] ) diff --git a/tests/llm_translation/test_unit_test_bedrock_invoke.py b/tests/llm_translation/test_unit_test_bedrock_invoke.py index 14f08c759c5..39f02263f03 100644 --- a/tests/llm_translation/test_unit_test_bedrock_invoke.py +++ b/tests/llm_translation/test_unit_test_bedrock_invoke.py @@ -59,7 +59,7 @@ def test_transform_request_invalid_provider(bedrock_transformer): """Test request transformation with invalid provider""" messages = [{"role": "user", "content": "Hello"}] - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Bedrock Invoke HTTPX: Unknown provider=None') as exc_info: bedrock_transformer.transform_request( model="invalid.model", messages=messages, diff --git a/tests/local_testing/test_auth_utils.py b/tests/local_testing/test_auth_utils.py index 9aecb7e10e4..88e8c02a606 100644 --- a/tests/local_testing/test_auth_utils.py +++ b/tests/local_testing/test_auth_utils.py @@ -264,10 +264,6 @@ def test_get_end_user_id_from_request_body_backwards_compatibility(): ["gpt-3.5-turbo", "gpt-4o-mini-general-deployment"], ), ({"model": "gpt-3.5-turbo"}, "gpt-3.5-turbo"), - ( - {"model": "gpt-3.5-turbo, gpt-4o-mini-general-deployment"}, - ["gpt-3.5-turbo", "gpt-4o-mini-general-deployment"], - ), ], ) def test_get_model_from_request(request_data, expected_model): diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index edf847f4cef..8dd90cbfb37 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -1433,7 +1433,7 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model): sync_stream=sync_mode, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='litellm\\.BadRequestError: OpenAIException - Invalid value') as exc_info: await _call_with_bad_role() assert exc_info.value.code == "invalid_value" diff --git a/tests/local_testing/test_file_types.py b/tests/local_testing/test_file_types.py index db83ba0e74b..7fda81ebd45 100644 --- a/tests/local_testing/test_file_types.py +++ b/tests/local_testing/test_file_types.py @@ -23,13 +23,13 @@ class TestFileConsts: def test_get_file_extension_from_mime_type(self): assert get_file_extension_from_mime_type("audio/aac") == "aac" assert get_file_extension_from_mime_type("application/pdf") == "pdf" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Unknown extension for mime type: application'): get_file_extension_from_mime_type("application/unknown") def test_get_file_type_from_extension(self): assert get_file_type_from_extension("aac") == FileType.AAC assert get_file_type_from_extension("pdf") == FileType.PDF - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Unknown file type for extension: unknown'): get_file_type_from_extension("unknown") def test_get_file_extension_for_file_type(self): diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index 385be25fb07..cef05050ac9 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -134,7 +134,6 @@ def test_get_model_info_bedrock_region(): "ft:gpt-3.5-turbo:my-org:custom_suffix:id", "ft:gpt-4-0613:my-org:custom_suffix:id", "ft:davinci-002:my-org:custom_suffix:id", - "ft:gpt-4-0613:my-org:custom_suffix:id", "ft:babbage-002:my-org:custom_suffix:id", "gpt-35-turbo", "ada", diff --git a/tests/local_testing/test_router_budget_limiter.py b/tests/local_testing/test_router_budget_limiter.py index 48915137138..4ef99ec8c12 100644 --- a/tests/local_testing/test_router_budget_limiter.py +++ b/tests/local_testing/test_router_budget_limiter.py @@ -160,7 +160,7 @@ async def test_provider_budgets_e2e_test_expect_to_fail(): await asyncio.sleep(2.5) for _ in range(3): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Exceeded budget for provider") as exc_info: await router.acompletion( messages=[{"role": "user", "content": "Hello, how are you?"}], model="anthropic/claude-sonnet-4-5-20250929", @@ -594,7 +594,7 @@ async def test_deployment_budgets_e2e_test_expect_to_fail(): await asyncio.sleep(2.5) for _ in range(3): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Exceeded budget for deployment") as exc_info: await router.acompletion( messages=[{"role": "user", "content": "Hello, how are you?"}], model="openai/gpt-4o-mini", @@ -646,7 +646,7 @@ async def test_tag_budgets_e2e_test_expect_to_fail(): await asyncio.sleep(2.5) for _ in range(3): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match=f"Exceeded budget for tag='{TAG_NAME}'") as exc_info: await router.acompletion( messages=[{"role": "user", "content": "Hello, how are you?"}], model="openai/gpt-4o-mini", diff --git a/tests/local_testing/test_router_fallbacks.py b/tests/local_testing/test_router_fallbacks.py index 15c6c5fec59..86dec406332 100644 --- a/tests/local_testing/test_router_fallbacks.py +++ b/tests/local_testing/test_router_fallbacks.py @@ -1430,7 +1430,7 @@ async def test_router_fallbacks_default_and_model_specific_fallbacks(sync_mode): messages=[{"role": "user", "content": "Hey, how's it going?"}], ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='litellm\\.AuthenticationError: AuthenticationError') as exc_info: await _call_bad_model() assert isinstance( exc_info.value, litellm.AuthenticationError diff --git a/tests/logging_callback_tests/test_standard_logging_payload.py b/tests/logging_callback_tests/test_standard_logging_payload.py index d13cdf1337a..6a632c32fc2 100644 --- a/tests/logging_callback_tests/test_standard_logging_payload.py +++ b/tests/logging_callback_tests/test_standard_logging_payload.py @@ -293,7 +293,7 @@ def test_cleanup_timestamps(): assert all(isinstance(x, float) for x in result) # Test invalid input - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="start_time is required, got=invalid of type "): StandardLoggingPayloadSetup.cleanup_timestamps( "invalid", end_float, completion_float ) diff --git a/tests/multi_instance_e2e_tests/test_update_team_e2e.py b/tests/multi_instance_e2e_tests/test_update_team_e2e.py index dfbfbd310ee..13091fd3df6 100644 --- a/tests/multi_instance_e2e_tests/test_update_team_e2e.py +++ b/tests/multi_instance_e2e_tests/test_update_team_e2e.py @@ -143,7 +143,7 @@ async def test_team_blocking_behavior_multi_instance(): assert team_info_4001["blocked"] is True, "Team should be blocked after update" # 8. Make a chat completion request on port 4000 with a new prompt; expect it to be blocked. - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match="(?i)blocked") as excinfo: await chat_completion_on_port( session, key=key, @@ -157,7 +157,7 @@ async def test_team_blocking_behavior_multi_instance(): ), f"Expected error indicating team blocked, got: {error_msg}" # 9. Make a chat completion request on port 4000 with a new prompt; expect it to be blocked. - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match="(?i)blocked") as excinfo: await chat_completion_on_port( session, key=key, @@ -171,7 +171,7 @@ async def test_team_blocking_behavior_multi_instance(): ), f"Expected error indicating team blocked, got: {error_msg}" # 9. Repeat the chat completion request with another new prompt; expect it to be blocked. - with pytest.raises(Exception) as excinfo_second: + with pytest.raises(Exception, match="(?i)blocked") as excinfo_second: await chat_completion_on_port( session, key=key, diff --git a/tests/ocr_tests/test_ocr_azure_document_intelligence.py b/tests/ocr_tests/test_ocr_azure_document_intelligence.py index 5736bd797e3..e6a2e5e5735 100644 --- a/tests/ocr_tests/test_ocr_azure_document_intelligence.py +++ b/tests/ocr_tests/test_ocr_azure_document_intelligence.py @@ -101,7 +101,7 @@ class TestAzureDocumentIntelligencePagesParam: cfg.map_ocr_params({"pages": [True, False]}, {}, "prebuilt-layout") def test_map_ocr_params_unsupported_type_raises(self, cfg): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='based, Mistral-style\\) or a string like'): cfg.map_ocr_params({"pages": 5}, {}, "prebuilt-layout") def test_get_complete_url_appends_pages_query(self, cfg): diff --git a/tests/otel_tests/test_e2e_model_access.py b/tests/otel_tests/test_e2e_model_access.py index 5b5f2a89c8d..e5e93c0b179 100644 --- a/tests/otel_tests/test_e2e_model_access.py +++ b/tests/otel_tests/test_e2e_model_access.py @@ -3,6 +3,7 @@ import asyncio import aiohttp import json from httpx import AsyncClient +from openai import PermissionDeniedError from typing import Any, Optional, List, Literal @@ -134,7 +135,7 @@ async def test_model_access_update(): await mock_chat_completion(session=session, key=key, model="openai/gpt-5.5") # Should fail with gpt-5-mini - with pytest.raises(Exception) as exc_info: + with pytest.raises(PermissionDeniedError) as exc_info: await mock_chat_completion( session=session, key=key, model="openai/gpt-5-mini" ) @@ -157,7 +158,7 @@ async def test_model_access_update(): ) # Non-OpenAI model should still fail - with pytest.raises(Exception) as exc_info: + with pytest.raises(PermissionDeniedError) as exc_info: await mock_chat_completion( session=session, key=key, model="anthropic/claude-2" ) @@ -254,7 +255,7 @@ async def test_team_model_access_update(): await mock_chat_completion(session=session, key=key, model="openai/gpt-5.5") # Should fail with gpt-5-mini - with pytest.raises(Exception) as exc_info: + with pytest.raises(PermissionDeniedError) as exc_info: await mock_chat_completion( session=session, key=key, model="openai/gpt-5-mini" ) @@ -279,7 +280,7 @@ async def test_team_model_access_update(): ) # Non-OpenAI model should still fail - with pytest.raises(Exception) as exc_info: + with pytest.raises(PermissionDeniedError) as exc_info: await mock_chat_completion( session=session, key=key, model="anthropic/claude-2" ) diff --git a/tests/proxy_admin_ui_tests/test_key_management.py b/tests/proxy_admin_ui_tests/test_key_management.py index 4c5a045509a..7e8494b77fc 100644 --- a/tests/proxy_admin_ui_tests/test_key_management.py +++ b/tests/proxy_admin_ui_tests/test_key_management.py @@ -1340,6 +1340,6 @@ async def test_team_model_alias(prisma_client, requested_model, should_pass): }, "Expected model aliases to be present" else: # Verify the key fails with non-aliased models - with pytest.raises(Exception) as exc_info: + with pytest.raises(ProxyException) as exc_info: await user_api_key_auth(request=request, api_key=f"Bearer {generated_key}") assert exc_info.value.type == ProxyErrorTypes.key_model_access_denied diff --git a/tests/proxy_admin_ui_tests/test_role_based_access.py b/tests/proxy_admin_ui_tests/test_role_based_access.py index 9398428bd67..f9506fb694b 100644 --- a/tests/proxy_admin_ui_tests/test_role_based_access.py +++ b/tests/proxy_admin_ui_tests/test_role_based_access.py @@ -9,7 +9,7 @@ from litellm._uuid import uuid from datetime import datetime from dotenv import load_dotenv -from fastapi import Request +from fastapi import HTTPException, Request from fastapi.routing import APIRoute load_dotenv() @@ -530,7 +530,7 @@ async def test_user_role_permissions(prisma_client, route, user_role, expected_r print(f"Auth passed as expected for {route} with role {user_role}") else: # Should raise an error - with pytest.raises(Exception) as exc_info: + with pytest.raises((ProxyException, HTTPException)) as exc_info: await user_api_key_auth(request=request, api_key=bearer_token) print(f"Auth failed as expected for {route} with role {user_role}") print(f"Error message: {str(exc_info.value)}") diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/proxy_unit_tests/test_auth_checks.py index ffcbe472be7..ef3cbd0ae95 100644 --- a/tests/proxy_unit_tests/test_auth_checks.py +++ b/tests/proxy_unit_tests/test_auth_checks.py @@ -173,7 +173,7 @@ async def test_can_key_call_model(model, expect_to_work): if expect_to_work: await can_key_call_model(**args) else: - with pytest.raises(Exception) as e: + with pytest.raises(Exception, match='key not allowed to access model\\. This key can only access') as e: await can_key_call_model(**args) print(e) @@ -958,7 +958,7 @@ async def test_can_key_call_model_with_aliases(model, alias_map, expect_to_work) llm_router=router, ) else: - with pytest.raises(Exception) as e: + with pytest.raises(Exception, match='key not allowed to access model\\. This key can only access') as e: await can_key_call_model( model=model, llm_model_list=llm_model_list, diff --git a/tests/proxy_unit_tests/test_jwt.py b/tests/proxy_unit_tests/test_jwt.py index 4db47a1cde4..686d7021257 100644 --- a/tests/proxy_unit_tests/test_jwt.py +++ b/tests/proxy_unit_tests/test_jwt.py @@ -1583,7 +1583,7 @@ async def test_auth_jwt_mismatched_key_fails(monkeypatch): h = JWTHandler() with patch.object(h, "get_public_key", new=AsyncMock(return_value=rsa_jwk)): - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='Validation fails: Expecting a PEM-formatted key\\.') as exc: await h.auth_jwt(token) assert "Validation fails" in str(exc.value) @@ -1826,7 +1826,7 @@ async def test_multi_issuer_jwt_unknown_issuer_without_global_jwks_rejected( kid="issuer-key", ) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='Missing JWT Public Key URL from environment\\.') as exc: await jwt_handler.auth_jwt(token=token) assert "Missing JWT Public Key URL" in str(exc.value) @@ -1857,7 +1857,7 @@ async def test_multi_issuer_jwt_rejects_wrong_audience(monkeypatch): kid="issuer-key", ) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match="Validation fails: Audience doesn't match") as exc: await jwt_handler.auth_jwt(token=token) assert "Validation fails" in str(exc.value) @@ -1900,7 +1900,7 @@ async def test_multi_issuer_jwt_same_kid_does_not_cross_issuer_keys(monkeypatch) kid=shared_kid, ) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='Validation fails: Signature verification failed') as exc: await jwt_handler.auth_jwt(token=token) assert "Validation fails" in str(exc.value) @@ -1953,7 +1953,7 @@ def test_multi_issuer_jwt_requires_audience_unless_explicitly_disabled( issuer = "https://issuer.example.com" jwks_url = f"{issuer}/keys" - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='must configure audience or set') as exc: LiteLLM_JWTAuth( issuers=[ { diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 00ed6d13f63..de2a9282300 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -1026,7 +1026,7 @@ def test_enforced_params_check( from litellm.proxy.litellm_pre_call_utils import _enforced_params_check if expected_error: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='in request body\\. This is a required param'): _enforced_params_check( request_body=request_body, general_settings=general_settings, @@ -2626,7 +2626,7 @@ async def test_during_call_hook_parallel_execution_with_error(): try: litellm.callbacks = [FailingGuardrail()] - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Guardrail violation detected!') as exc_info: await proxy_logging.during_call_hook( data={ "model": "gpt-4", diff --git a/tests/proxy_unit_tests/test_update_spend.py b/tests/proxy_unit_tests/test_update_spend.py index 0d1d6dcf3c6..6b8973fbad2 100644 --- a/tests/proxy_unit_tests/test_update_spend.py +++ b/tests/proxy_unit_tests/test_update_spend.py @@ -166,7 +166,7 @@ async def test_update_spend_logs_non_connection_error(): prisma_client.db.litellm_spendlogs.create_many = create_many_mock # Execute and verify it raises immediately without retrying - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Unexpected database error') as exc_info: await update_spend(prisma_client, None, proxy_logging_obj) # Verify error message diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 1fef0f01df8..f81578dbd99 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -90,7 +90,7 @@ def test_routing_strategy_init_invalid_strategy(model_list): router = Router(model_list=model_list) # Test common mistake: "simple" instead of "simple-shuffle" - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="usage-based-routing', 'provider-budget-routing'\\]\\. Check") as exc_info: router.routing_strategy_init( routing_strategy="simple", routing_strategy_args={} ) @@ -106,7 +106,7 @@ def test_routing_strategy_init_invalid_strategy(model_list): assert "Router SDK" in error_msg # Test completely invalid strategy - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="usage-based-routing', 'provider-budget-routing'\\]\\. Check") as exc_info: router.routing_strategy_init( routing_strategy="not-a-real-strategy", routing_strategy_args={} ) diff --git a/tests/store_model_in_db_tests/test_mcp_servers.py b/tests/store_model_in_db_tests/test_mcp_servers.py index e9c26221580..735d5d71ad3 100644 --- a/tests/store_model_in_db_tests/test_mcp_servers.py +++ b/tests/store_model_in_db_tests/test_mcp_servers.py @@ -471,7 +471,7 @@ def test_validate_mcp_server_name_direct(): validate_mcp_server_name("valid name") # Test that invalid names with hyphens raise exceptions - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Server name cannot contain '-'\\. Use an alternative") as exc_info: validate_mcp_server_name("invalid-name") assert "cannot contain" in str(exc_info.value) diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py index 40439a78a49..5fe4b217e4f 100644 --- a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py +++ b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py @@ -104,7 +104,7 @@ async def test_send_email_missing_api_key(): try: logger = SendGridEmailLogger() - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='SENDGRID_API_KEY is not set'): await logger.send_email( from_email="test@example.com", to_email=["recipient@example.com"], diff --git a/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py b/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py index 6cc31f991a3..fcd03e77aa2 100644 --- a/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py +++ b/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py @@ -471,7 +471,7 @@ async def test_afile_content_error_reports_unified_id_not_provider_uri(): mock_router.get_deployment_credentials_with_provider = MagicMock(return_value=None) mock_router.afile_content = AsyncMock(side_effect=Exception("deployment failed")) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='LiteLLM Managed File object with') as exc_info: await managed_files.afile_content( file_id=unified_file_id, litellm_parent_otel_span=None, diff --git a/tests/test_litellm/integrations/cloudzero/test_cloudzero_database.py b/tests/test_litellm/integrations/cloudzero/test_cloudzero_database.py index 89a5028011c..7f930f90247 100644 --- a/tests/test_litellm/integrations/cloudzero/test_cloudzero_database.py +++ b/tests/test_litellm/integrations/cloudzero/test_cloudzero_database.py @@ -51,7 +51,7 @@ async def test_get_usage_data_rejects_invalid_limit(monkeypatch: pytest.MonkeyPa """limit must coerce to int or raise ValueError before hitting the DB.""" db, query_mock = _setup_db(monkeypatch, []) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='limit must be an integer'): await db.get_usage_data(limit="invalid") assert query_mock.await_count == 0 diff --git a/tests/test_litellm/integrations/cloudzero/test_cz_stream_api.py b/tests/test_litellm/integrations/cloudzero/test_cz_stream_api.py index 440ce39e021..a715116e5ee 100644 --- a/tests/test_litellm/integrations/cloudzero/test_cz_stream_api.py +++ b/tests/test_litellm/integrations/cloudzero/test_cz_stream_api.py @@ -108,7 +108,7 @@ class TestCloudZeroStreamer: """Test _parse_and_convert_timestamp method with invalid timestamp.""" streamer = CloudZeroStreamer("test-key", "test-connection") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Could not parse timestamp 'invalid-timestamp': Invalid"): streamer._parse_and_convert_timestamp("invalid-timestamp") def test_prepare_batch_payload(self): diff --git a/tests/test_litellm/integrations/focus/test_focus_database.py b/tests/test_litellm/integrations/focus/test_focus_database.py index d77af2dd170..5c13665f1f1 100644 --- a/tests/test_litellm/integrations/focus/test_focus_database.py +++ b/tests/test_litellm/integrations/focus/test_focus_database.py @@ -68,7 +68,7 @@ async def test_should_accept_string_timestamps(monkeypatch: pytest.MonkeyPatch): async def test_should_reject_invalid_limit(monkeypatch: pytest.MonkeyPatch): db, query_mock = _setup_db(monkeypatch, []) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='limit must be an integer'): await db.get_usage_data(limit="invalid") assert query_mock.await_count == 0 diff --git a/tests/test_litellm/integrations/focus/test_s3_destination.py b/tests/test_litellm/integrations/focus/test_s3_destination.py index f915b2c56a3..8e54b561f82 100644 --- a/tests/test_litellm/integrations/focus/test_s3_destination.py +++ b/tests/test_litellm/integrations/focus/test_s3_destination.py @@ -20,7 +20,7 @@ def _window(freq: str = "hourly", hour: int = 5) -> FocusTimeWindow: def test_should_require_bucket_name(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='bucket_name must be provided for S'): FocusS3Destination(prefix="focus", config={}) diff --git a/tests/test_litellm/integrations/gitlab/test_gitlab_client.py b/tests/test_litellm/integrations/gitlab/test_gitlab_client.py index 4556950cd3e..529868ca06a 100644 --- a/tests/test_litellm/integrations/gitlab/test_gitlab_client.py +++ b/tests/test_litellm/integrations/gitlab/test_gitlab_client.py @@ -95,9 +95,9 @@ def enc_project(p): # how client encodes project in urls # Constructor / config tests # ----------------------------- def test_init_requires_project_and_token(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='project and access_token are required'): GitLabClient({"project": "p"}) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='project and access_token are required'): GitLabClient({"access_token": "t"}) @@ -127,7 +127,7 @@ def test_set_ref_updates_effective_ref(): c = make_client(branch="main") c.set_ref("feature/x") assert c.ref == "feature/x" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='ref must be a non-empty string'): c.set_ref("") @@ -193,12 +193,12 @@ def test_get_file_content_permission_errors_are_mapped(): raw_url = f"https://gitlab.example.com/api/v4/projects/{enc_project('group/sub/repo')}/repository/files/secure%2Ffile.prompt/raw?ref=main" # raise_for_status will be called, so return 403 response (not an exception from transport) c.http_handler.routes[raw_url] = FakeResponse(status_code=403) - with pytest.raises(Exception) as ei: + with pytest.raises(Exception, match="Check your GitLab permissions for project 'group") as ei: c.get_file_content("secure/file.prompt") assert "Access denied" in str(ei.value) c.http_handler.routes[raw_url] = FakeResponse(status_code=401) - with pytest.raises(Exception) as ei2: + with pytest.raises(Exception, match='Authentication failed\\. Check your GitLab token and') as ei2: c.get_file_content("secure/file.prompt") assert "Authentication failed" in str(ei2.value) diff --git a/tests/test_litellm/integrations/levo/test_levo.py b/tests/test_litellm/integrations/levo/test_levo.py index 98b0327dbf2..903be644671 100644 --- a/tests/test_litellm/integrations/levo/test_levo.py +++ b/tests/test_litellm/integrations/levo/test_levo.py @@ -198,7 +198,7 @@ class TestLevoIntegration(unittest.TestCase): """Test health check returns unhealthy status when required vars are missing.""" # Try to create logger without required env vars # This should fail during config, but we can test health check logic - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='LEVOAI_API_KEY environment variable is required for Levo'): LevoLogger.get_levo_config() @patch.dict( diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py b/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py index e1b8e4b5721..b810ffdc6be 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py @@ -554,7 +554,7 @@ def test_token_type_rejected_from_either_list(attributes, monkeypatch): recorder rather than silently ignored, so the misconfig is caught at all.""" recorder = _recorder(monkeypatch, attributes) kwargs, response_obj, start, end = _build_call() - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='otel\\.attributes: gen_ai\\.token\\.type is a structural') as exc_info: recorder.record(kwargs, response_obj, start, end) # The dedicated discriminator guard, not the generic unknown-name path: assert # the specific reason so dropping that guard (and falling through to "unknown diff --git a/tests/test_litellm/integrations/test_langfuse.py b/tests/test_litellm/integrations/test_langfuse.py index 6e57a36c5b6..3c7dd51bff8 100644 --- a/tests/test_litellm/integrations/test_langfuse.py +++ b/tests/test_litellm/integrations/test_langfuse.py @@ -1163,7 +1163,7 @@ def test_max_langfuse_clients_limit(): assert litellm.initialized_langfuse_clients == 2 # Third client should fail with exception - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Max langfuse clients reached') as exc_info: logger3 = LangFuseLogger( langfuse_public_key="test_key_3", langfuse_secret="test_secret_3", diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index fffbc884782..08d8c17cc2e 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1169,7 +1169,7 @@ def test_bedrock_image_processor_content_type_fallback_failure(): # Test with URL without recognizable extension image_url = "https://example.com/unknown-file" - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match='Unable to determine content type from URL: https') as excinfo: BedrockImageProcessor._post_call_image_processing(mock_response, image_url) assert "Unable to determine content type" in str(excinfo.value) diff --git a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py index 0dca4f3a1b1..956f86a9292 100644 --- a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py +++ b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py @@ -110,7 +110,7 @@ def test_top_level_kwargs_overrides_metadata_slots(): def test_env_reference_at_top_level_raises_with_guidance(): kwargs = {"langfuse_public_key": "os.environ/LANGFUSE_PUBLIC_KEY"} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="Callback param 'langfuse_public_key' \\(from request body\\)") as exc_info: initialize_standard_callback_dynamic_params(kwargs) message = str(exc_info.value) @@ -127,7 +127,7 @@ def test_env_reference_in_metadata_raises_with_guidance(): } } - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="Callback param 'langsmith_api_key' \\(from metadata\\) contains") as exc_info: initialize_standard_callback_dynamic_params(kwargs) message = str(exc_info.value) diff --git a/tests/test_litellm/litellm_core_utils/test_llm_judge.py b/tests/test_litellm/litellm_core_utils/test_llm_judge.py index 5c092caa7c3..a0a2311914b 100644 --- a/tests/test_litellm/litellm_core_utils/test_llm_judge.py +++ b/tests/test_litellm/litellm_core_utils/test_llm_judge.py @@ -27,7 +27,7 @@ def test_parse_json_verdict_tolerates_fences_and_prose(raw, expected): def test_parse_json_verdict_rejects_non_object(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='judge response is not a JSON object'): parse_json_verdict('["not", "an", "object"]') with pytest.raises((json.JSONDecodeError, ValueError)): parse_json_verdict("no json here at all") diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 456d4db4afe..fbdfcac1adc 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -982,7 +982,7 @@ async def test_bedrock_validation_error_raises_directly(logging_obj: Logging): make_call=_raise_400, ) - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match='litellm\\.BadRequestError: BedrockException') as excinfo: await response.__anext__() assert not isinstance(excinfo.value, MidStreamFallbackError) assert getattr(excinfo.value, "status_code", None) == 400 @@ -2722,7 +2722,7 @@ def test_dispatch_text_completion_codestral_requires_string( is a programming error and must surface loudly.""" initialized_custom_stream_wrapper.custom_llm_provider = "text-completion-codestral" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="chunk is not a string: \\{'not': 'a string'\\}"): _run_dispatch(initialized_custom_stream_wrapper, {"not": "a string"}) diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 3c33ee13c3f..eec4b307c87 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -763,24 +763,6 @@ class TestTokenizerSelection(unittest.TestCase): ], } ], - [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "These are some sample images from a movie. Based on these images, what do you think the tone of the movie is?", - }, - { - "type": "text", - "image_url": { - "url": "https://gratisography.com/wp-content/uploads/2024/11/gratisography-augmented-reality-800x525.jpg", - "detail": "high", - }, - }, - ], - } - ], ], ) def test_bad_input_token_counter(model, messages): @@ -1174,7 +1156,7 @@ def test_count_content_list_rejects_unknown_type(): """ from litellm.litellm_core_utils.token_counter import _count_content_list - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Error getting number of tokens from content list: Invalid') as exc_info: _count_content_list( count_function=len, content_list=[{"type": "totally_unknown_block"}], diff --git a/tests/test_litellm/litellm_core_utils/test_url_utils.py b/tests/test_litellm/litellm_core_utils/test_url_utils.py index cef09f3f2b0..751b548adcd 100644 --- a/tests/test_litellm/litellm_core_utils/test_url_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_url_utils.py @@ -100,12 +100,12 @@ class TestEncodeUrlPathSegment: @pytest.mark.parametrize("value", ["", ".", "..", None]) def test_rejects_empty_and_dot_segments(self, value): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="resource_id (is required|cannot be a dot path segment)"): encode_url_path_segment(value, field_name="resource_id") @pytest.mark.parametrize("value", ["../model", "model/../other", "/model"]) def test_rejects_dot_segments_in_multi_segment_paths(self, value): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="model (is required|cannot be a dot path segment)"): encode_url_path_segments(value, field_name="model") diff --git a/tests/test_litellm/llms/aiml/image_generation/test_aiml_image_generation_transformation.py b/tests/test_litellm/llms/aiml/image_generation/test_aiml_image_generation_transformation.py index 7485f2121df..dd74379a883 100644 --- a/tests/test_litellm/llms/aiml/image_generation/test_aiml_image_generation_transformation.py +++ b/tests/test_litellm/llms/aiml/image_generation/test_aiml_image_generation_transformation.py @@ -113,7 +113,7 @@ def test_flux_style_request_still_remaps_to_legacy_fields(): def test_openai_style_unsupported_param_raises_without_drop_params(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Supported parameters are'): AimlImageGenerationConfig().map_openai_params( non_default_params={"image_size": {"width": 1024, "height": 1024}}, optional_params={}, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py index 654b0097546..f3cb2956aeb 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py @@ -61,7 +61,7 @@ def test_anthropic_messages_handler_skips_the_gateway_on_recursion(): "litellm.llms.anthropic.experimental_pass_through.messages.mcp_handler.anthropic_messages_with_mcp", new=AsyncMock(return_value={"routed": True}), ) as routed: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='anthropic_messages_handler is not implemented for sync calls'): anthropic_messages_handler( max_tokens=100, messages=[{"role": "user", "content": "hi"}], @@ -80,7 +80,7 @@ def test_anthropic_messages_handler_leaves_native_tools_alone(): "litellm.llms.anthropic.experimental_pass_through.messages.mcp_handler.anthropic_messages_with_mcp", new=AsyncMock(return_value={"routed": True}), ) as routed: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='anthropic_messages_handler is not implemented for sync calls'): anthropic_messages_handler( max_tokens=100, messages=[{"role": "user", "content": "hi"}], diff --git a/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py b/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py index ffabce6e00c..602cbf68f3f 100644 --- a/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py +++ b/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py @@ -16,7 +16,7 @@ class TestAzureAIRerankConfigGetCompleteUrl: self.model = "azure_ai/cohere-rerank-v3-english" def test_api_base_required(self): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Azure AI API Base is required\\. api_base=None\\. Set in') as exc_info: self.config.get_complete_url(api_base=None, model=self.model) assert "api_base=None" in str(exc_info.value) @@ -31,7 +31,7 @@ class TestAzureAIRerankConfigGetCompleteUrl: ], ) def test_api_base_requires_scheme(self, api_base): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Azure AI API Base must be an absolute URL including scheme') as exc_info: self.config.get_complete_url(api_base=api_base, model=self.model) error_message = str(exc_info.value).lower() diff --git a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py index ee50b9db015..e8964910c69 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py @@ -2,17 +2,20 @@ import os import sys from unittest.mock import AsyncMock, MagicMock +import httpx import pytest sys.path.insert( 0, os.path.abspath("../../../../..") ) # Adds the parent directory to the system path +import litellm from litellm.llms.bedrock.chat.invoke_handler import ( AWSEventStreamDecoder, make_call, make_sync_call, ) +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler def test_transform_thinking_blocks_with_redacted_content(): @@ -293,3 +296,50 @@ def test_make_sync_call_honors_explicit_stream_chunk_size(): response.iter_bytes.assert_called_once_with(chunk_size=2048) + +def test_invoke_streaming_forwards_bedrock_response_headers(): + response = MagicMock() + response.status_code = 200 + response.iter_bytes = MagicMock(return_value=iter([])) + response.headers = httpx.Headers({"x-amzn-requestid": "req-789"}) + client = HTTPHandler() + client.post = MagicMock(return_value=response) + + stream = litellm.completion( + model="bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "hi"}], + stream=True, + client=client, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + + assert stream._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-789" + + +@pytest.mark.asyncio +async def test_async_invoke_streaming_forwards_bedrock_response_headers(): + async def _no_bytes(chunk_size=None): + return + yield b"" + + response = MagicMock() + response.status_code = 200 + response.aiter_bytes = _no_bytes + response.headers = httpx.Headers({"x-amzn-requestid": "req-987"}) + client = AsyncHTTPHandler() + client.post = AsyncMock(return_value=response) + + stream = await litellm.acompletion( + model="bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "hi"}], + stream=True, + client=client, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + + assert stream._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-987" + diff --git a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py index cfe9930e76e..b9f8283b78e 100644 --- a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py +++ b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py @@ -1944,7 +1944,7 @@ def test_role_assumption_access_denied_raises_when_different_role(): with patch.object( base_aws_llm, "_is_already_running_as_role", return_value=False ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='An error occurred \\(AccessDenied\\) when calling the') as exc_info: base_aws_llm._auth_with_aws_role( aws_access_key_id=None, aws_secret_access_key=None, @@ -1969,7 +1969,7 @@ def test_role_assumption_non_access_denied_error_propagated(): ) with patch("boto3.client", return_value=mock_sts_client): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='An error occurred \\(MalformedPolicyDocument\\) when calling') as exc_info: base_aws_llm._auth_with_aws_role( aws_access_key_id=None, aws_secret_access_key=None, diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index 8281f3387d9..28c8e5c7ed6 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -109,7 +109,7 @@ class TestBedrockMantleResponsesURL: monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) monkeypatch.delenv("AWS_REGION", raising=False) cfg = BedrockMantleResponsesAPIConfig() - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="api\\.aws\\.attacker\\.example/'\\. Region names must contain only"): cfg.get_complete_url( api_base=None, litellm_params={ @@ -1418,7 +1418,7 @@ class TestBedrockMantleResponsesSigV4: signer.get_credentials = MagicMock(side_effect=NoCredentialsError()) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, @@ -1448,7 +1448,7 @@ class TestBedrockMantleResponsesSigV4: signer.get_credentials = MagicMock(side_effect=cred_error) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index 275fb460b9f..07910b0b56f 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -107,7 +107,7 @@ class TestBedrockMantleConfig: monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) monkeypatch.delenv("AWS_REGION", raising=False) cfg = BedrockMantleChatConfig() - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="api\\.aws\\.attacker\\.example/'\\. Region names must contain only"): cfg._get_openai_compatible_provider_info( None, None, @@ -416,7 +416,7 @@ class TestBedrockMantleChatAuth: signer.get_credentials = MagicMock(side_effect=NoCredentialsError()) cfg = BedrockMantleChatConfig(aws_signer=signer) - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, diff --git a/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py b/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py index e2421437720..94b8c51dd52 100644 --- a/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py +++ b/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py @@ -38,7 +38,7 @@ class TestBytezChatConfig: config = BytezChatConfig() headers = {} - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match='Missing api_key, make sure you pass in your api key') as excinfo: config.validate_environment( headers=headers, model=TEST_MODEL, diff --git a/tests/test_litellm/llms/chat/test_converse_handler.py b/tests/test_litellm/llms/chat/test_converse_handler.py index 2a3db5982ef..6f8a2788c38 100644 --- a/tests/test_litellm/llms/chat/test_converse_handler.py +++ b/tests/test_litellm/llms/chat/test_converse_handler.py @@ -1,14 +1,16 @@ +import json import os import sys -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock +import httpx import pytest import litellm from litellm.llms.bedrock.chat import BedrockConverseLLM from litellm.llms.bedrock.chat.converse_handler import make_sync_call from litellm.llms.bedrock.common_utils import _get_all_bedrock_regions -from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler sys.path.insert( 0, os.path.abspath("../../../../..") @@ -202,6 +204,104 @@ def test_make_sync_call_honors_explicit_stream_chunk_size(): response.iter_bytes.assert_called_once_with(chunk_size=2048) +def _converse_response_body() -> dict: + return { + "output": {"message": {"role": "assistant", "content": [{"text": "hi"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + } + + +def test_converse_completion_forwards_bedrock_response_headers(): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json = MagicMock(return_value=_converse_response_body()) + mock_response.text = json.dumps(_converse_response_body()) + mock_response.headers = httpx.Headers({"x-amzn-requestid": "req-123"}) + client = HTTPHandler() + client.post = MagicMock(return_value=mock_response) + + response = litellm.completion( + model="bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "hi"}], + client=client, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + + assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-123" + + +def test_converse_streaming_forwards_bedrock_response_headers(): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.iter_bytes = MagicMock(return_value=iter([])) + mock_response.headers = httpx.Headers({"x-amzn-requestid": "req-456"}) + client = HTTPHandler() + client.post = MagicMock(return_value=mock_response) + + response = litellm.completion( + model="bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "hi"}], + stream=True, + client=client, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + + assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-456" + + +@pytest.mark.asyncio +async def test_async_converse_completion_forwards_bedrock_response_headers(): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json = MagicMock(return_value=_converse_response_body()) + mock_response.text = json.dumps(_converse_response_body()) + mock_response.headers = httpx.Headers({"x-amzn-requestid": "req-abc"}) + client = AsyncHTTPHandler() + client.post = AsyncMock(return_value=mock_response) + + response = await litellm.acompletion( + model="bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "hi"}], + client=client, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + + assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-abc" + + +@pytest.mark.asyncio +async def test_async_converse_streaming_forwards_bedrock_response_headers(): + async def _no_bytes(chunk_size=None): + return + yield b"" + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.aiter_bytes = _no_bytes + mock_response.headers = httpx.Headers({"x-amzn-requestid": "req-def"}) + client = AsyncHTTPHandler() + client.post = AsyncMock(return_value=mock_response) + + response = await litellm.acompletion( + model="bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "hi"}], + stream=True, + client=client, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + + assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-def" + + def test_completion_plumbs_stream_chunk_size_through_converse(): iter_bytes_spy = _stream_completion_with_spied_iter_bytes( model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index c568b82ebba..c87abbd8bc4 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -2369,3 +2369,79 @@ async def test_async_anthropic_messages_handler_carries_deployment_vertex_locati unconfigured_deployment = await logging_obj_after_handler(GenericLiteLLMParams()) assert "vertex_location" not in unconfigured_deployment.litellm_params + + +_GENERIC_STREAM_SSE = ( + b'data: {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,' + b'"model":"test-model","choices":[{"index":0,"delta":{"content":"hi"},' + b'"finish_reason":null}]}\n\n' + b"data: [DONE]\n\n" +) + + +def _generic_stream_upstream_response() -> httpx.Response: + return httpx.Response( + 200, + headers={ + "x-request-id": "generic-req-123", + "x-ratelimit-remaining-requests": "42", + }, + content=_GENERIC_STREAM_SSE, + request=httpx.Request("POST", "https://fake-vllm.test/v1/chat/completions"), + ) + + +def test_generic_http_handler_sync_streaming_forwards_provider_response_headers(): + """ + Regression test for the generic BaseLLMHTTPHandler streaming path used by + ~30 providers (deepseek, groq, hosted_vllm, databricks, openrouter, ...). + + The sync `completion()` streaming branch builds the CustomStreamWrapper from + `make_sync_call`, which returns the upstream response headers alongside the + stream. Those headers must reach the caller as `llm_provider-*` entries in + `_hidden_params["additional_headers"]`, which is what the proxy merges into + the client-facing response headers. + """ + mock_client = Mock(spec=HTTPHandler) + mock_client.post = Mock(return_value=_generic_stream_upstream_response()) + + response = litellm.completion( + model="hosted_vllm/test-model", + messages=[{"role": "user", "content": "Hello"}], + api_base="https://fake-vllm.test/v1", + api_key="sk-test", + stream=True, + client=mock_client, + ) + + additional_headers = response._hidden_params["additional_headers"] + assert additional_headers["llm_provider-x-request-id"] == "generic-req-123" + assert additional_headers["llm_provider-x-ratelimit-remaining-requests"] == "42" + + assert "".join([chunk.choices[0].delta.content or "" for chunk in response]) == "hi" + + +@pytest.mark.asyncio +async def test_generic_http_handler_async_streaming_forwards_provider_response_headers(): + """ + Companion to the sync test above for `acompletion_stream_function`, which + builds its CustomStreamWrapper from `make_async_call_stream_helper`. + """ + mock_client = AsyncMock(spec=AsyncHTTPHandler) + mock_client.post = AsyncMock(return_value=_generic_stream_upstream_response()) + + response = await litellm.acompletion( + model="hosted_vllm/test-model", + messages=[{"role": "user", "content": "Hello"}], + api_base="https://fake-vllm.test/v1", + api_key="sk-test", + stream=True, + client=mock_client, + ) + + additional_headers = response._hidden_params["additional_headers"] + assert additional_headers["llm_provider-x-request-id"] == "generic-req-123" + assert additional_headers["llm_provider-x-ratelimit-remaining-requests"] == "42" + + collected = [chunk async for chunk in response] + assert "".join([chunk.choices[0].delta.content or "" for chunk in collected]) == "hi" diff --git a/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank_transformation.py b/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank_transformation.py index a5411078cf7..ae3c166e7aa 100644 --- a/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank_transformation.py +++ b/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank_transformation.py @@ -258,7 +258,7 @@ class TestDeepinfraRerankTransform: status_code = 401 headers = {"content-type": "application/json"} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Authentication failed') as exc_info: self.config.get_error_class(error_message, status_code, headers) # The method should raise a BaseLLMException @@ -271,7 +271,7 @@ class TestDeepinfraRerankTransform: status_code = 404 headers = {"content-type": "application/json"} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Model not found') as exc_info: self.config.get_error_class(error_message, status_code, headers) # Should extract the nested error message @@ -284,7 +284,7 @@ class TestDeepinfraRerankTransform: status_code = 503 headers = {"content-type": "application/json"} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Service unavailable') as exc_info: self.config.get_error_class(error_message, status_code, headers) # Should extract the string detail @@ -296,7 +296,7 @@ class TestDeepinfraRerankTransform: status_code = 500 headers = {"content-type": "application/json"} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Invalid JSON error message') as exc_info: self.config.get_error_class(error_message, status_code, headers) # Should use the original error message when JSON parsing fails diff --git a/tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py b/tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py index 593593bfa73..c0f74eff51b 100644 --- a/tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py +++ b/tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py @@ -113,7 +113,7 @@ def test_response_format_is_ignored(): def test_unsupported_param_raises_without_drop_params(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Supported parameters are \\['n', 'response_format', 'size'\\]\\."): FalAINanoBananaConfig().map_openai_params( non_default_params={"style": "vivid"}, optional_params={}, diff --git a/tests/test_litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py b/tests/test_litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py index 4dc467575a0..bf40abd7016 100644 --- a/tests/test_litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py +++ b/tests/test_litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py @@ -44,7 +44,7 @@ class TestFeatherlessAIConfig: """Test error handling when API key is missing""" config = FeatherlessAIConfig() - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match='Missing Featherless AI API Key') as excinfo: config.validate_environment( headers={}, model="featherless-ai/Qwerky-72B", @@ -112,7 +112,7 @@ class TestFeatherlessAIConfig: "tool_choice": {"type": "function", "function": {"name": "get_weather"}} } optional_params = {} - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match="litellm\\.UnsupportedParamsError: Featherless AI doesn't") as excinfo: config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, @@ -138,7 +138,7 @@ class TestFeatherlessAIConfig: assert "tools" not in result # Test with tools and drop_params=False - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match="litellm\\.UnsupportedParamsError: Featherless AI doesn't") as excinfo: config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, diff --git a/tests/test_litellm/llms/fireworks_ai/rerank/test_fireworks_ai_rerank_transformation.py b/tests/test_litellm/llms/fireworks_ai/rerank/test_fireworks_ai_rerank_transformation.py index 30bf5860dee..521ea4f8263 100644 --- a/tests/test_litellm/llms/fireworks_ai/rerank/test_fireworks_ai_rerank_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/rerank/test_fireworks_ai_rerank_transformation.py @@ -301,7 +301,7 @@ class TestFireworksAIRerankTransform: mock_logging = MagicMock() model_response = RerankResponse() - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Failed to parse response: Invalid JSON: line') as exc_info: self.config.transform_rerank_response( model=self.model, raw_response=mock_response, diff --git a/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py b/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py index 9b57e1991de..bd9b7006e58 100644 --- a/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py +++ b/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py @@ -244,7 +244,7 @@ class TestGeminiImageEditTransformation: def test_transform_image_edit_request_without_image_raises(self) -> None: optional_params = {} - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Gemini image edit requires at least one image\\.'): self.config.transform_image_edit_request( model=self.model, prompt=self.prompt, diff --git a/tests/test_litellm/llms/gemini/test_gemini_client_setup.py b/tests/test_litellm/llms/gemini/test_gemini_client_setup.py index 51c6fedf5b8..48b010aca48 100644 --- a/tests/test_litellm/llms/gemini/test_gemini_client_setup.py +++ b/tests/test_litellm/llms/gemini/test_gemini_client_setup.py @@ -28,7 +28,7 @@ def test_gemini_completion_no_api_key(): del os.environ[key] # Test without mock_response to ensure actual API key validation - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='in _complete_vertex_ai_beta') as exc_info: completion( model="gemini/gemini-1.5-flash", messages=[{"role": "user", "content": "Test message"}], @@ -60,7 +60,7 @@ def test_gemini_completion_no_api_key_with_mock(): with patch("litellm.get_secret") as mock_get_secret: mock_get_secret.return_value = None - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='in _complete_vertex_ai_beta') as exc_info: completion( model="gemini/gemini-1.5-flash", messages=[{"role": "user", "content": "Test message"}], diff --git a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py index 6425e815db0..e6e6aa946d5 100644 --- a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py @@ -109,7 +109,7 @@ class TestHostedVLLMRerankTransform: ) assert url2 == "https://api.example.com/rerank" # Raises if api_base is None - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='api_base must be provided for Hosted VLLM rerank'): self.config.get_complete_url(None, self.model) def test_transform_response(self): diff --git a/tests/test_litellm/llms/langflow/chat/test_langflow_chat_transformation.py b/tests/test_litellm/llms/langflow/chat/test_langflow_chat_transformation.py index b3c4e0f1858..0c241add77b 100644 --- a/tests/test_litellm/llms/langflow/chat/test_langflow_chat_transformation.py +++ b/tests/test_litellm/llms/langflow/chat/test_langflow_chat_transformation.py @@ -46,7 +46,7 @@ def test_langflow_config_get_complete_url(): def test_langflow_config_get_complete_url_requires_api_base(): config = LangFlowConfig() - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='api_base is required for LangFlow\\. Set it via'): config.get_complete_url( api_base=None, api_key=None, diff --git a/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py b/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py index 7f00f53c451..fbcec3d4d2e 100644 --- a/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py +++ b/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py @@ -154,7 +154,7 @@ class TestModelScopeImageGenerationTransformation: mock_get_secret.return_value = None headers = {} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='MODELSCOPE_API_KEY is not set\\. Please set it via') as exc_info: self.config.validate_environment( headers=headers, model=self.model, @@ -367,7 +367,7 @@ class TestModelScopeImageGenerationTransformation: model_response = ImageResponse(data=[]) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='litellm\\.BadRequestError: ModelScope error: Invalid prompt') as exc_info: self.config.transform_image_generation_response( model=self.model, raw_response=mock_response, @@ -393,7 +393,7 @@ class TestModelScopeImageGenerationTransformation: model_response = ImageResponse(data=[]) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='litellm\\.InternalServerError: Error parsing ModelScope') as exc_info: self.config.transform_image_generation_response( model=self.model, raw_response=mock_response, diff --git a/tests/test_litellm/llms/novita/chat/test_novita_chat_transformation.py b/tests/test_litellm/llms/novita/chat/test_novita_chat_transformation.py index 7a00b361252..ade5e4176e8 100644 --- a/tests/test_litellm/llms/novita/chat/test_novita_chat_transformation.py +++ b/tests/test_litellm/llms/novita/chat/test_novita_chat_transformation.py @@ -47,7 +47,7 @@ class TestNovitaConfig: """Test error handling when API key is missing""" config = NovitaConfig() - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match='Missing Novita AI API Key - A call is being made to novita') as excinfo: config.validate_environment( headers={}, model="novita/meta-llama/llama-3.3-70b-instruct", diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py index 8be0780d86f..5aa96a66d2d 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -98,7 +98,7 @@ class TestOCIChatConfig: config = OCIChatConfig() headers = {} - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match='Missing required parameters: oci_user, oci_fingerprint') as excinfo: config.validate_environment( headers=headers, model=TEST_MODEL, @@ -272,7 +272,7 @@ class TestOCIChatConfig: "oci_serving_mode": "INVALID_MODE", } - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match="kwarg `oci_serving_mode` must be either 'ON_DEMAND' or") as excinfo: config.transform_request( model=TEST_MODEL_NAME, messages=TEST_MESSAGES, # type: ignore @@ -892,7 +892,7 @@ class TestOCISignerSupport: optional_params = {"oci_signer": MockSigner(), "method": "INVALID"} - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match='Unsupported HTTP method: INVALID') as excinfo: config.sign_request( headers={}, optional_params=optional_params, @@ -1604,7 +1604,7 @@ class TestOCIKeyNormalization: # We can't fully test signing without a real key, but we can verify # the error message indicates the key was processed (not a type error) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='why-can-t-i-import-my-pem-file for more details\\.') as exc_info: sign_with_manual_credentials( headers={}, optional_params=optional_params, @@ -1630,7 +1630,7 @@ class TestOCIKeyNormalization: "oci_key": crlf_pem, } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='why-can-t-i-import-my-pem-file for more details\\.') as exc_info: sign_with_manual_credentials( headers={}, optional_params=optional_params, @@ -1692,7 +1692,7 @@ class TestOCIValidateEnvironment: def test_missing_required_credentials_raises_error(self, config): """Test that missing required credentials raise an error.""" - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Missing required parameters: oci_user, oci_fingerprint') as exc_info: config.validate_environment( headers={}, model="oci/xai.grok-3", @@ -1875,7 +1875,7 @@ class TestOCIImageUrlTransformation: } ] - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Prop `image_url` must be a string or an object with a `url`') as exc_info: adapt_messages_to_generic_oci_standard(messages) assert "image_url" in str(exc_info.value) @@ -1899,7 +1899,7 @@ class TestOCIImageUrlTransformation: } ] - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Prop `image_url` must be a string or an object with a `url`') as exc_info: adapt_messages_to_generic_oci_standard(messages) assert "image_url" in str(exc_info.value) diff --git a/tests/test_litellm/llms/openai_like/test_cognition_provider.py b/tests/test_litellm/llms/openai_like/test_cognition_provider.py index c358d178f60..5c71b60e08a 100644 --- a/tests/test_litellm/llms/openai_like/test_cognition_provider.py +++ b/tests/test_litellm/llms/openai_like/test_cognition_provider.py @@ -111,33 +111,51 @@ class TestCognitionProviderIdentity: class TestCognitionCostTracking: @pytest.mark.parametrize( - "model, input_cost, output_cost", + "model, input_cost, output_cost, cache_read_cost", [ - ("cognition/swe-1.6", 5e-07, 2.5e-06), - ("cognition/swe-1.7", 2.5e-06, 1.25e-05), + ("cognition/swe-1.6", 5e-07, 2.5e-06, 2e-07), + ("cognition/swe-1.7", 5e-07, 2.5e-06, 2e-07), + ("cognition/swe-1.7-lightning", 2.5e-06, 1.25e-05, 1e-06), ], ) - def test_cost_map_entries(self, model: str, input_cost: float, output_cost: float): + def test_cost_map_entries(self, model: str, input_cost: float, output_cost: float, cache_read_cost: float): info = litellm.get_model_info(model=model) assert info["litellm_provider"] == "cognition" assert info["mode"] == "chat" assert info["input_cost_per_token"] == input_cost assert info["output_cost_per_token"] == output_cost + assert info["cache_read_input_token_cost"] == cache_read_cost - def test_cost_differs_from_openai_pricing(self): + @pytest.mark.parametrize( + "model, expected_prompt_cost, expected_completion_cost", + [ + ("cognition/swe-1.7", 0.5, 2.5), + ("cognition/swe-1.7-lightning", 2.5, 12.5), + ], + ) + def test_cost_differs_from_openai_pricing( + self, model: str, expected_prompt_cost: float, expected_completion_cost: float + ): """A cognition-prefixed model must never be priced off an OpenAI cost entry.""" from litellm.cost_calculator import cost_per_token prompt_cost, completion_cost = cost_per_token( - model="cognition/swe-1.7", + model=model, prompt_tokens=1_000_000, completion_tokens=1_000_000, custom_llm_provider="cognition", ) - assert prompt_cost == pytest.approx(2.5) - assert completion_cost == pytest.approx(12.5) + assert prompt_cost == pytest.approx(expected_prompt_cost) + assert completion_cost == pytest.approx(expected_completion_cost) + + def test_lightning_is_five_times_the_standard_tier(self): + standard = litellm.get_model_info(model="cognition/swe-1.7") + lightning = litellm.get_model_info(model="cognition/swe-1.7-lightning") + + assert lightning["input_cost_per_token"] == pytest.approx(standard["input_cost_per_token"] * 5) + assert lightning["output_cost_per_token"] == pytest.approx(standard["output_cost_per_token"] * 5) def test_supported_endpoints_matrix(self): matrix = json.loads((Path(litellm.__file__).parent / "provider_endpoints_support_backup.json").read_text()) @@ -170,6 +188,30 @@ class TestCognitionRouting: mock_response="hello from swe", ) + usage = response.usage + expected = usage.prompt_tokens * 5e-07 + usage.completion_tokens * 2.5e-06 + assert response._hidden_params["response_cost"] == pytest.approx(expected) + + @pytest.mark.asyncio + async def test_router_spend_uses_the_lightning_entry_for_lightning(self): + """The Lightning tier is its own model, costed off its own entry.""" + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "swe-lightning", + "litellm_params": {"model": "cognition/swe-1.7-lightning", "api_key": "sk-test"}, + } + ] + ) + + response = await router.acompletion( + model="swe-lightning", + messages=[{"role": "user", "content": "hi"}], + mock_response="hello from swe lightning", + ) + usage = response.usage expected = usage.prompt_tokens * 2.5e-06 + usage.completion_tokens * 1.25e-05 assert response._hidden_params["response_cost"] == pytest.approx(expected) diff --git a/tests/test_litellm/llms/pg_vector/vector_stores/test_pg_vector_transformation.py b/tests/test_litellm/llms/pg_vector/vector_stores/test_pg_vector_transformation.py index 56953a574d6..1d44b2bc278 100644 --- a/tests/test_litellm/llms/pg_vector/vector_stores/test_pg_vector_transformation.py +++ b/tests/test_litellm/llms/pg_vector/vector_stores/test_pg_vector_transformation.py @@ -42,7 +42,7 @@ class TestPGVectorStoreConfig: litellm_params = GenericLiteLLMParams() headers = {} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='PG Vector API key is required\\. Set PG_VECTOR_API_KEY') as exc_info: config.validate_environment(headers, litellm_params) assert "PG Vector API key is required" in str(exc_info.value) @@ -84,7 +84,7 @@ class TestPGVectorStoreConfig: config = PGVectorStoreConfig() litellm_params = {} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='PG Vector API base URL is required\\. Set') as exc_info: config.get_complete_url(None, litellm_params) assert "PG Vector API base URL is required" in str(exc_info.value) diff --git a/tests/test_litellm/llms/recraft/image_edit/test_recraft_image_edit_transformation.py b/tests/test_litellm/llms/recraft/image_edit/test_recraft_image_edit_transformation.py index 0acabd05805..47811321133 100644 --- a/tests/test_litellm/llms/recraft/image_edit/test_recraft_image_edit_transformation.py +++ b/tests/test_litellm/llms/recraft/image_edit/test_recraft_image_edit_transformation.py @@ -167,7 +167,7 @@ class TestRecraftImageEditTransformation: mock_response.status_code = 500 mock_response.headers = {} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Error transforming image edit response: Invalid JSON: line') as exc_info: self.config.transform_image_edit_response( model=self.model, raw_response=mock_response, diff --git a/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py b/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py index 70311201969..ccc72dde7b8 100644 --- a/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py +++ b/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py @@ -64,7 +64,7 @@ class TestRecraftImageGenerationTransformation: non_default_params = {"n": 2, "unsupported_param": "value"} optional_params = {} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Supported parameters are') as exc_info: self.config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, @@ -171,7 +171,7 @@ class TestRecraftImageGenerationTransformation: mock_get_secret.return_value = None headers = {} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='RECRAFT_API_KEY is not set') as exc_info: self.config.validate_environment( headers=headers, model=self.model, @@ -248,7 +248,7 @@ class TestRecraftImageGenerationTransformation: model_response = ImageResponse(data=[]) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Error transforming image generation response: Invalid JSON') as exc_info: self.config.transform_image_generation_response( model=self.model, raw_response=mock_response, diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_chat_transformation.py b/tests/test_litellm/llms/sagemaker/test_sagemaker_chat_transformation.py index 54e6f95c795..da6caca4f05 100644 --- a/tests/test_litellm/llms/sagemaker/test_sagemaker_chat_transformation.py +++ b/tests/test_litellm/llms/sagemaker/test_sagemaker_chat_transformation.py @@ -19,6 +19,8 @@ from unittest.mock import MagicMock import httpx import pytest +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.sagemaker.chat.transformation import SagemakerChatConfig @@ -233,3 +235,85 @@ def test_decoder_reassembles_frames_across_arbitrary_byte_boundaries(split_size) ] assert texts == [f"token{i} " for i in range(len(frames))] + + +_INFERENCE_COMPONENT_HEADER = "X-Amzn-SageMaker-Inference-Component" + +_STUB_COMPLETION_RESPONSE = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1700000000, + "model": "served-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +class _RequestCapturingHTTPHandler(HTTPHandler): + """Injected transport that records exactly what sagemaker_chat put on the wire.""" + + def __init__(self) -> None: + super().__init__() + self.request_headers: dict[str, str] = {} + self.request_body: dict = {} + + def post(self, url: str, headers=None, data=None, **kwargs) -> httpx.Response: + self.request_headers = dict(headers or {}) + self.request_body = json.loads(data) + return httpx.Response(200, json=_STUB_COMPLETION_RESPONSE, request=httpx.Request("POST", url)) + + +def _invoke_sagemaker_chat(monkeypatch, **extra_params) -> _RequestCapturingHTTPHandler: + """Drive one sagemaker_chat completion against an injected transport. + + A Bedrock API key short-circuits SigV4 inside `BaseAWSLLM._sign_request`, which would hide + whether the inference-component header is really covered by the signature, so it is cleared. + """ + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + client = _RequestCapturingHTTPHandler() + litellm.completion( + model="sagemaker_chat/my-endpoint", + messages=[{"role": "user", "content": "hi"}], + aws_access_key_id="AKIATESTTESTTESTTEST", + aws_secret_access_key="test-secret-key", + aws_region_name="us-east-1", + client=client, + **extra_params, + ) + return client + + +def test_model_id_is_sent_as_a_signed_inference_component_header(monkeypatch): + """`model_id` names an inference component and must reach SageMaker as a signed header. + + Endpoints backed by inference components reject any request without + `X-Amzn-SageMaker-Inference-Component` with HTTP 400 INFERENCE_COMPONENT_NAME_MISSING, so the + header has to be built before `sign_request` runs and end up inside SignedHeaders. + """ + client = _invoke_sagemaker_chat(monkeypatch, model_id="my-inference-component") + + assert client.request_headers[_INFERENCE_COMPONENT_HEADER] == "my-inference-component" + assert "x-amzn-sagemaker-inference-component" in client.request_headers["Authorization"] + + +def test_no_inference_component_header_when_model_id_is_unset(monkeypatch): + """Plain endpoints must not receive the header at all, not even an empty one.""" + client = _invoke_sagemaker_chat(monkeypatch) + + assert not any(name.lower() == _INFERENCE_COMPONENT_HEADER.lower() for name in client.request_headers) + + +def test_hf_model_name_becomes_the_body_model(monkeypatch): + """`hf_model_name` names the served model, and containers that validate the body's `model` + 404 on the endpoint name, so it has to replace it rather than ride along as an extra field.""" + client = _invoke_sagemaker_chat(monkeypatch, hf_model_name="org/served-model") + + assert client.request_body["model"] == "org/served-model" + assert "hf_model_name" not in client.request_body + + +def test_body_model_stays_the_endpoint_name_when_hf_model_name_is_unset(monkeypatch): + """Without `hf_model_name` the body must keep the model it has today.""" + client = _invoke_sagemaker_chat(monkeypatch) + + assert client.request_body["model"] == "my-endpoint" diff --git a/tests/test_litellm/llms/stability/image_generation/test_stability_image_generation.py b/tests/test_litellm/llms/stability/image_generation/test_stability_image_generation.py index 6f1a04e78d3..c5b3c8fbdc5 100644 --- a/tests/test_litellm/llms/stability/image_generation/test_stability_image_generation.py +++ b/tests/test_litellm/llms/stability/image_generation/test_stability_image_generation.py @@ -83,7 +83,7 @@ class TestStabilityImageGenerationConfig: non_default_params = {"unsupported_param": "value"} optional_params = {} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="Supported parameters are \\['n', 'size',") as exc_info: self.config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, @@ -168,7 +168,7 @@ class TestStabilityImageGenerationConfig: def test_validate_environment_raises_without_api_key(self): """Test that validate_environment raises error without API key""" - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='STABILITY_API_KEY is not set\\. Please set it via') as exc_info: self.config.validate_environment( headers={}, model="stability/sd3", @@ -251,7 +251,7 @@ class TestStabilityImageGenerationConfig: model_response = ImageResponse(data=[]) mock_logging = MagicMock() - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Content was filtered by Stability AI safety systems') as exc_info: self.config.transform_image_generation_response( model="stability/sd3", raw_response=mock_response, diff --git a/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py b/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py index 2dcccb8ea7e..69afbb416aa 100644 --- a/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py +++ b/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py @@ -697,7 +697,7 @@ class TestErrorHandling: } } mock_response = _make_mock_response(body, status_code=400) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='TinyFish Search: query is required\\. See https') as exc_info: config.transform_search_response( raw_response=mock_response, logging_obj=None ) @@ -713,7 +713,7 @@ class TestErrorHandling: mock_response = _make_mock_response( body, status_code=429, headers={"Retry-After": "60"} ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='TinyFish Search: rate limit exceeded\\. See https') as exc_info: config.transform_search_response( raw_response=mock_response, logging_obj=None ) @@ -728,7 +728,7 @@ class TestErrorHandling: config = TinyfishSearchConfig() body = {"errors": [{"code": "10000", "message": "Internal"}]} mock_response = _make_mock_response(body, status_code=502) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='TinyFish Search') as exc_info: config.transform_search_response( raw_response=mock_response, logging_obj=None ) @@ -742,7 +742,7 @@ class TestErrorHandling: mock_response = _make_mock_response( json_data=None, status_code=502, text="Bad Gateway" ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='TinyFish Search: Bad Gateway<') as exc_info: config.transform_search_response( raw_response=mock_response, logging_obj=None ) @@ -756,7 +756,7 @@ class TestErrorHandling: mock_response = _make_mock_response( json_data=None, status_code=200, text="not json" ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='TinyFish Search: Expected JSON response, got: not json\\.') as exc_info: config.transform_search_response( raw_response=mock_response, logging_obj=None ) @@ -785,7 +785,7 @@ class TestErrorHandling: # check TinyFish's schema, not their own input. config = TinyfishSearchConfig() mock_response = _make_mock_response({"query": "x"}) # no `results` key - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='validation error for SearchResponse') as exc_info: config.transform_search_response( raw_response=mock_response, logging_obj=None ) diff --git a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_integration.py b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_integration.py index 272565990bd..8f9acafa49d 100644 --- a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_integration.py +++ b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_integration.py @@ -159,7 +159,7 @@ class TestVertexAIFilesIntegration: # This test ensures the type annotations and error messages include vertex_ai # Test that calling with unsupported provider raises appropriate error - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="unsupported_provider' is not a valid LlmProviders") as exc_info: litellm.file_content( file_id="test-file-id", custom_llm_provider="unsupported_provider", # This should fail diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index b83d4742b64..c189cdd0ea7 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -33,7 +33,7 @@ def test_validate_vertex_location_accepts_valid(location): ["attacker.example/", "evil.com#", "us.attacker.example", "us/../..", "US", "us_central1", "-us", "", None], ) def test_validate_vertex_location_rejects_invalid(location): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="vertex_location is required|Invalid vertex_location format"): validate_vertex_location(location) diff --git a/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py b/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py index 891d1c15c61..7922331d19f 100644 --- a/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py +++ b/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py @@ -137,7 +137,7 @@ class TestVolcengineResponsesAPITransformation: monkeypatch.delenv("ARK_API_KEY", raising=False) monkeypatch.delenv("VOLCENGINE_API_KEY", raising=False) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Volcengine API key is required\\. Set ARK_API_KEY /'): config.validate_environment(headers={}, model="volcengine/demo", litellm_params={}) def test_unsupported_params_are_dropped_with_extra_body(self): diff --git a/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py b/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py index 07298f03f86..1670dac0e9d 100644 --- a/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py +++ b/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py @@ -202,7 +202,7 @@ def test_volcengine_embedding_error_scenarios(): k: v for k, v in scenario.items() if k != "expected_error_pattern" } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match=f"(?i){scenario['expected_error_pattern']}") as exc_info: litellm.embedding(input=["test"], **test_params) # Verify error message contains expected pattern diff --git a/tests/test_litellm/llms/voyage/rerank/test_voyage_rerank_transformation.py b/tests/test_litellm/llms/voyage/rerank/test_voyage_rerank_transformation.py index 8f99609e3f5..f466b7e19b5 100644 --- a/tests/test_litellm/llms/voyage/rerank/test_voyage_rerank_transformation.py +++ b/tests/test_litellm/llms/voyage/rerank/test_voyage_rerank_transformation.py @@ -227,7 +227,7 @@ class TestVoyageRerankTransform: mock_logging = MagicMock() model_response = RerankResponse() - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Unauthorized') as exc_info: self.config.transform_rerank_response( model=self.model, raw_response=mock_response, @@ -248,7 +248,7 @@ class TestVoyageRerankTransform: mock_logging = MagicMock() model_response = RerankResponse() - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Failed to parse response: Invalid JSON response') as exc_info: self.config.transform_rerank_response( model=self.model, raw_response=mock_response, diff --git a/tests/test_litellm/llms/voyage/test_voyage_multimodal_embedding.py b/tests/test_litellm/llms/voyage/test_voyage_multimodal_embedding.py index f283e7fe0df..f3e6885cbe6 100644 --- a/tests/test_litellm/llms/voyage/test_voyage_multimodal_embedding.py +++ b/tests/test_litellm/llms/voyage/test_voyage_multimodal_embedding.py @@ -195,7 +195,7 @@ class TestVoyageMultimodalEmbeddings: monkeypatch.setattr(module, "get_secret_str", lambda name: None) config = VoyageMultimodalEmbeddingConfig() - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Voyage API key is required for multimodal embeddings\\. Set') as exc_info: config.validate_environment( {}, "voyage-multimodal-3.5", [], {}, {}, api_key=None ) @@ -207,7 +207,7 @@ class TestVoyageMultimodalEmbeddings: ) config = VoyageMultimodalEmbeddingConfig() - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Voyage multimodal embeddings require a non-empty') as exc_info: config._normalize_content_item({"type": "image_url", "image_url": {}}) assert "image_url" in str(exc_info.value) diff --git a/tests/test_litellm/llms/xai/test_xai_key_fallback.py b/tests/test_litellm/llms/xai/test_xai_key_fallback.py index 4c769c572ac..ec3eb83309c 100644 --- a/tests/test_litellm/llms/xai/test_xai_key_fallback.py +++ b/tests/test_litellm/llms/xai/test_xai_key_fallback.py @@ -168,7 +168,7 @@ def test_responses_config_raises_when_no_key_is_available(monkeypatch): monkeypatch.setattr(litellm, "api_key", None) monkeypatch.delenv("XAI_API_KEY", raising=False) - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='XAI API key is required\\. Set api_key, litellm\\.xai_key') as exc_info: XAIResponsesAPIConfig().validate_environment({}, "xai/grok-3-mini", None) error_message = str(exc_info.value) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 0209abee510..d5936b2ae86 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -8185,7 +8185,7 @@ class TestGetUserObjectPermission: return_value=None, ), ): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="user 'human-dangling' names object_permission_id"): await MCPRequestHandler._get_user_object_permission(auth) async def test_no_user_id_places_no_ceiling(self): diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 34852850de6..b4d3782ba43 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -2695,13 +2695,6 @@ async def test_token_endpoint_respects_x_forwarded_host(): "443", "https://internal.local", ), - ( - "http://localhost:4000/", - "https", - "proxy.example.com", - "8443", - "https://proxy.example.com:8443", - ), ( "http://localhost:4000/", "https", diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_short_mcp_tool_prefix.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_short_mcp_tool_prefix.py index 6e3ac014840..941e5deee93 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_short_mcp_tool_prefix.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_short_mcp_tool_prefix.py @@ -79,7 +79,7 @@ class TestShortPrefixHelpers: assert compute_short_server_prefix("abc") != compute_short_server_prefix("abd") def test_short_prefix_requires_server_id(self): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='compute_short_server_prefix requires a non-empty server_id'): compute_short_server_prefix("") def test_flag_defaults_to_false(self): diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 6b40fa1b324..762d2cbf3c7 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -268,7 +268,7 @@ def test_get_experimental_ui_login_jwt_auth_token_invalid( invalid_sso_user_defined_values, ): """Test generating JWT token with missing user role""" - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='User role is required for experimental UI login') as exc_info: ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token( invalid_sso_user_defined_values ) @@ -883,7 +883,7 @@ async def test_get_user_object_wraps_db_outage_as_valueerror_preserving_context( mock_cache.async_set_cache = AsyncMock() with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="User doesn't exist in db\\.") as exc_info: await get_user_object( user_id="outage-contract-probe-user", prisma_client=mock_prisma_client, diff --git a/tests/test_litellm/proxy/auth/test_auth_hot_path_network_requests.py b/tests/test_litellm/proxy/auth/test_auth_hot_path_network_requests.py index 22752f767ce..6b2d2babedc 100644 --- a/tests/test_litellm/proxy/auth/test_auth_hot_path_network_requests.py +++ b/tests/test_litellm/proxy/auth/test_auth_hot_path_network_requests.py @@ -1,521 +1,521 @@ -""" -Test to count and track the number of network requests (DB queries, cache lookups) -made on the hot path for keys that have team_id and user_id attached. - -This test ensures we don't regress on the number of network requests made during -request authentication, which directly impacts proxy latency. - -The hot path covers auth functions called on every LLM API request: -- get_key_object: lookup the API key -- get_team_object: lookup the team (for keys with team_id) -- get_user_object: lookup the user (for keys with user_id) -- get_team_membership: lookup team member budget (when team_member_spend set) - -Each function does: cache read -> (on miss) DB query -> cache write. -We count these to catch regressions in the number of network requests. - -NOTE: This test does NOT require proxy extras (apscheduler, etc.) because -it tests at the auth_checks level, not the full proxy_server level. -""" - -import os -import sys -import time -from typing import Any, Dict, List, Optional -from unittest.mock import AsyncMock, MagicMock - -import pytest - -sys.path.insert(0, os.path.abspath("../../..")) - -from litellm.caching.dual_cache import DualCache -from litellm.caching.in_memory_cache import InMemoryCache -from litellm.proxy._types import ( - LiteLLM_TeamTableCachedObj, - LiteLLM_UserTable, - LitellmUserRoles, - UserAPIKeyAuth, - LiteLLM_TeamMembership, - hash_token, -) -from litellm.proxy.auth.auth_checks import ( - get_key_object, - get_team_membership, - get_team_object, - get_user_object, -) - - -class CacheCallTracker: - """ - Tracks cache read/write operations by wrapping DualCache methods. - This is used to count network-level operations on the hot path. - """ - - def __init__(self): - self.cache_reads: List[Dict[str, Any]] = [] - self.cache_writes: List[Dict[str, Any]] = [] - self.db_queries: List[Dict[str, Any]] = [] - - def get_summary(self) -> Dict[str, Any]: - return { - "total_cache_reads": len(self.cache_reads), - "total_cache_writes": len(self.cache_writes), - "total_db_queries": len(self.db_queries), - "total_network_requests": len(self.cache_reads) - + len(self.cache_writes) - + len(self.db_queries), - "cache_read_keys": [r["key"] for r in self.cache_reads], - "cache_write_keys": [w["key"] for w in self.cache_writes], - "db_query_details": self.db_queries, - } - - -def _wrap_cache_with_tracker(cache: DualCache, tracker: CacheCallTracker) -> DualCache: - """Wrap a DualCache to track all reads and writes.""" - original_async_get = cache.async_get_cache - original_async_set = cache.async_set_cache - - async def tracked_async_get(key, *args, **kwargs): - result = await original_async_get(key, *args, **kwargs) - tracker.cache_reads.append( - {"key": key, "hit": result is not None, "method": "async_get_cache"} - ) - return result - - async def tracked_async_set(key, value, *args, **kwargs): - tracker.cache_writes.append({"key": key, "method": "async_set_cache"}) - return await original_async_set(key, value, *args, **kwargs) - - cache.async_get_cache = tracked_async_get - cache.async_set_cache = tracked_async_set - return cache - - -def _create_valid_token( - api_key: str, - team_id: str, - user_id: str, - has_team_member_spend: bool = False, - org_id: Optional[str] = None, -) -> UserAPIKeyAuth: - """Create a UserAPIKeyAuth with team_id and user_id set.""" - hashed = hash_token(api_key) - return UserAPIKeyAuth( - token=hashed, - api_key=api_key, - team_id=team_id, - user_id=user_id, - org_id=org_id, - models=["gpt-4", "gpt-3.5-turbo"], - max_budget=100.0, - spend=10.0, - team_spend=50.0, - team_max_budget=1000.0, - team_models=["gpt-4", "gpt-3.5-turbo"], - team_member_spend=5.0 if has_team_member_spend else None, - last_refreshed_at=time.time(), - user_role=LitellmUserRoles.INTERNAL_USER, - ) - - -def _create_team_object(team_id: str) -> LiteLLM_TeamTableCachedObj: - """Create a team table object for caching.""" - return LiteLLM_TeamTableCachedObj( - team_id=team_id, - models=["gpt-4", "gpt-3.5-turbo"], - max_budget=1000.0, - spend=50.0, - tpm_limit=10000, - rpm_limit=100, - last_refreshed_at=time.time(), - ) - - -def _create_user_object(user_id: str) -> LiteLLM_UserTable: - """Create a user table object for caching.""" - return LiteLLM_UserTable( - user_id=user_id, - max_budget=500.0, - spend=25.0, - models=["gpt-4"], - tpm_limit=5000, - rpm_limit=50, - user_role=LitellmUserRoles.INTERNAL_USER, - user_email="test@example.com", - ) - - -# ============================================================================ -# TEST: get_key_object cache behavior -# ============================================================================ - - -@pytest.mark.asyncio -async def test_get_key_object_warm_cache(): - """ - Test get_key_object with a warm cache - should hit cache, no DB query. - """ - api_key = "sk-test-key-warm" - team_id = "team-123" - user_id = "user-456" - hashed_token = hash_token(api_key) - - valid_token = _create_valid_token(api_key, team_id, user_id) - - # Create cache with pre-populated data - cache = DualCache(in_memory_cache=InMemoryCache()) - await cache.async_set_cache(key=hashed_token, value=valid_token) - - # Track cache operations - tracker = CacheCallTracker() - tracked_cache = _wrap_cache_with_tracker(cache, tracker) - - # Mock prisma client (should NOT be called for warm cache) - mock_prisma = MagicMock() - mock_prisma.get_data = AsyncMock() - - result = await get_key_object( - hashed_token=hashed_token, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - ) - - summary = tracker.get_summary() - - # Should have exactly 1 cache read - assert summary["total_cache_reads"] == 1 - assert hashed_token in summary["cache_read_keys"] - - # Prisma should NOT have been called - mock_prisma.get_data.assert_not_called() - - # Result should be the cached token - assert result.token == hashed_token - - -@pytest.mark.asyncio -async def test_get_key_object_cold_cache(): - """ - Test get_key_object with a cold cache - should miss cache, query DB. - """ - api_key = "sk-test-key-cold" - team_id = "team-123" - user_id = "user-456" - hashed_token = hash_token(api_key) - - valid_token = _create_valid_token(api_key, team_id, user_id) - - # Create empty cache - cache = DualCache(in_memory_cache=InMemoryCache()) - - tracker = CacheCallTracker() - tracked_cache = _wrap_cache_with_tracker(cache, tracker) - - # Mock prisma client to return token on DB query - mock_prisma = MagicMock() - mock_prisma.get_data = AsyncMock(return_value=valid_token) - - await get_key_object( - hashed_token=hashed_token, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - ) - - summary = tracker.get_summary() - - # Should have 1 cache read (miss) and at least 1 cache write (populate cache) - assert summary["total_cache_reads"] >= 1 - - # Prisma SHOULD have been called - mock_prisma.get_data.assert_called_once() - - -# ============================================================================ -# TEST: get_team_object cache behavior -# ============================================================================ - - -@pytest.mark.asyncio -async def test_get_team_object_warm_cache(): - """ - Test get_team_object with a warm cache - should hit cache, no DB query. - """ - team_id = "team-warm-123" - team_obj = _create_team_object(team_id) - - cache = DualCache(in_memory_cache=InMemoryCache()) - cache_key = f"team_id:{team_id}" - await cache.async_set_cache(key=cache_key, value=team_obj) - - tracker = CacheCallTracker() - tracked_cache = _wrap_cache_with_tracker(cache, tracker) - - mock_prisma = MagicMock() - mock_prisma.db = MagicMock() - mock_prisma.db.litellm_teamtable = MagicMock() - mock_prisma.db.litellm_teamtable.find_unique = AsyncMock() - - await get_team_object( - team_id=team_id, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - ) - - summary = tracker.get_summary() - - assert summary["total_cache_reads"] >= 1 - assert cache_key in summary["cache_read_keys"] - - # DB should NOT have been called - mock_prisma.db.litellm_teamtable.find_unique.assert_not_called() - - -# ============================================================================ -# TEST: get_user_object cache behavior -# ============================================================================ - - -@pytest.mark.asyncio -async def test_get_user_object_warm_cache(): - """ - Test get_user_object with a warm cache - should hit cache, no DB query. - """ - user_id = "user-warm-456" - user_obj = _create_user_object(user_id) - - cache = DualCache(in_memory_cache=InMemoryCache()) - await cache.async_set_cache(key=user_id, value=user_obj) - - tracker = CacheCallTracker() - tracked_cache = _wrap_cache_with_tracker(cache, tracker) - - mock_prisma = MagicMock() - mock_prisma.db = MagicMock() - mock_prisma.db.litellm_usertable = MagicMock() - mock_prisma.db.litellm_usertable.find_unique = AsyncMock() - - await get_user_object( - user_id=user_id, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - user_id_upsert=False, - ) - - summary = tracker.get_summary() - - assert summary["total_cache_reads"] >= 1 - assert user_id in summary["cache_read_keys"] - - # DB should NOT have been called - mock_prisma.db.litellm_usertable.find_unique.assert_not_called() - - -# ============================================================================ -# TEST: get_team_membership cache behavior -# ============================================================================ - - -@pytest.mark.asyncio -async def test_get_team_membership_warm_cache(): - """ - Test get_team_membership with a warm cache - should hit cache, no DB query. - """ - user_id = "user-tm-456" - team_id = "team-tm-123" - - membership_dict = { - "user_id": user_id, - "team_id": team_id, - "spend": 3.0, - "budget_id": None, - "litellm_budget_table": None, - } - - cache = DualCache(in_memory_cache=InMemoryCache()) - # Cache key format used by get_team_membership - cache_key = f"team_membership:{user_id}:{team_id}" - await cache.async_set_cache(key=cache_key, value=membership_dict) - - tracker = CacheCallTracker() - tracked_cache = _wrap_cache_with_tracker(cache, tracker) - - mock_prisma = MagicMock() - mock_prisma.db = MagicMock() - mock_prisma.db.litellm_teammembership = MagicMock() - mock_prisma.db.litellm_teammembership.find_unique = AsyncMock() - - await get_team_membership( - user_id=user_id, - team_id=team_id, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - ) - - summary = tracker.get_summary() - - assert summary["total_cache_reads"] >= 1 - assert cache_key in summary["cache_read_keys"] - - # DB should NOT have been called - mock_prisma.db.litellm_teammembership.find_unique.assert_not_called() - - -# ============================================================================ -# TEST: Document duplicate team membership cache key issue -# ============================================================================ - - -@pytest.mark.asyncio -async def test_team_membership_cache_key_duplication(): - """ - Document the team membership duplicate cache key issue: - - Team membership is queried via TWO different cache keys: - 1. "{team_id}_{user_id}" - used in user_api_key_auth.py:1048 - 2. "team_membership:{user_id}:{team_id}" - used in auth_checks.py:960 (get_team_membership) - - This test documents that both keys refer to the same data but use different - cache key formats, potentially leading to duplicate lookups. - """ - user_id = "user-dup-456" - team_id = "team-dup-123" - - # The two different cache keys used for the same data - key_format_1 = f"{team_id}_{user_id}" # user_api_key_auth format - key_format_2 = f"team_membership:{user_id}:{team_id}" # auth_checks format - - _ = { - "user_id": user_id, - "team_id": team_id, - "spend": 3.0, - } - - # Document that these are different keys - assert ( - key_format_1 != key_format_2 - ), "Cache keys should be different (this is the bug)" - - # Document that these are different keys - assert ( - key_format_1 != key_format_2 - ), "Cache keys should be different (this is the bug)" - - -# ============================================================================ -# TEST: Full hot path network count summary -# ============================================================================ - - -@pytest.mark.asyncio -async def test_full_hot_path_network_count(): - """ - Summary test that counts all network operations when processing - a request with a key that has team_id and user_id attached. - - This test verifies the baseline number of cache operations expected - on a fully warm cache path. - """ - api_key = "sk-test-full-path" - team_id = "team-full-123" - user_id = "user-full-456" - hashed_token = hash_token(api_key) - - # Create all objects - valid_token = _create_valid_token( - api_key, team_id, user_id, has_team_member_spend=True - ) - team_obj = _create_team_object(team_id) - user_obj = _create_user_object(user_id) - membership_data = LiteLLM_TeamMembership( - user_id=user_id, - team_id=team_id, - spend=3.0, - budget_id=None, - litellm_budget_table=None, - ) - - # Pre-populate cache with all data - cache = DualCache(in_memory_cache=InMemoryCache()) - await cache.async_set_cache(key=hashed_token, value=valid_token) - await cache.async_set_cache(key=f"team_id:{team_id}", value=team_obj) - await cache.async_set_cache(key=user_id, value=user_obj) - await cache.async_set_cache( - key=f"team_membership:{user_id}:{team_id}", value=membership_data.model_dump() - ) - await cache.async_set_cache( - key=f"{team_id}_{user_id}", value=membership_data.model_dump() - ) - - # Create tracker AFTER populating cache - tracker = CacheCallTracker() - tracked_cache = _wrap_cache_with_tracker(cache, tracker) - - # Mock prisma (should not be called on warm cache) - mock_prisma = MagicMock() - - # Call each function to simulate the hot path - await get_key_object( - hashed_token=hashed_token, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - ) - - await get_team_object( - team_id=team_id, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - ) - - await get_user_object( - user_id=user_id, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - user_id_upsert=False, - ) - - await get_team_membership( - user_id=user_id, - team_id=team_id, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - ) - - summary = tracker.get_summary() - - # Assertions for expected baseline - # On warm cache: 4 reads (key, team, user, team_membership) - assert ( - summary["total_cache_reads"] == 4 - ), f"Expected 4 cache reads on warm path, got {summary['total_cache_reads']}" - - # No DB queries on warm cache - assert ( - summary["total_db_queries"] == 0 - ), f"Expected 0 DB queries on warm path, got {summary['total_db_queries']}" - - # Total network requests should be exactly 4 on warm cache - assert ( - summary["total_network_requests"] == 4 - ), f"Expected 4 total network requests on warm path, got {summary['total_network_requests']}" +""" +Test to count and track the number of network requests (DB queries, cache lookups) +made on the hot path for keys that have team_id and user_id attached. + +This test ensures we don't regress on the number of network requests made during +request authentication, which directly impacts proxy latency. + +The hot path covers auth functions called on every LLM API request: +- get_key_object: lookup the API key +- get_team_object: lookup the team (for keys with team_id) +- get_user_object: lookup the user (for keys with user_id) +- get_team_membership: lookup team member budget (when team_member_spend set) + +Each function does: cache read -> (on miss) DB query -> cache write. +We count these to catch regressions in the number of network requests. + +NOTE: This test does NOT require proxy extras (apscheduler, etc.) because +it tests at the auth_checks level, not the full proxy_server level. +""" + +import os +import sys +import time +from typing import Any, Dict, List, Optional +from unittest.mock import AsyncMock, MagicMock + +import pytest + +sys.path.insert(0, os.path.abspath("../../..")) + +from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.proxy._types import ( + LiteLLM_TeamTableCachedObj, + LiteLLM_UserTable, + LitellmUserRoles, + UserAPIKeyAuth, + LiteLLM_TeamMembership, + hash_token, +) +from litellm.proxy.auth.auth_checks import ( + get_key_object, + get_team_membership, + get_team_object, + get_user_object, +) + + +class CacheCallTracker: + """ + Tracks cache read/write operations by wrapping DualCache methods. + This is used to count network-level operations on the hot path. + """ + + def __init__(self): + self.cache_reads: List[Dict[str, Any]] = [] + self.cache_writes: List[Dict[str, Any]] = [] + self.db_queries: List[Dict[str, Any]] = [] + + def get_summary(self) -> Dict[str, Any]: + return { + "total_cache_reads": len(self.cache_reads), + "total_cache_writes": len(self.cache_writes), + "total_db_queries": len(self.db_queries), + "total_network_requests": len(self.cache_reads) + + len(self.cache_writes) + + len(self.db_queries), + "cache_read_keys": [r["key"] for r in self.cache_reads], + "cache_write_keys": [w["key"] for w in self.cache_writes], + "db_query_details": self.db_queries, + } + + +def _wrap_cache_with_tracker(cache: DualCache, tracker: CacheCallTracker) -> DualCache: + """Wrap a DualCache to track all reads and writes.""" + original_async_get = cache.async_get_cache + original_async_set = cache.async_set_cache + + async def tracked_async_get(key, *args, **kwargs): + result = await original_async_get(key, *args, **kwargs) + tracker.cache_reads.append( + {"key": key, "hit": result is not None, "method": "async_get_cache"} + ) + return result + + async def tracked_async_set(key, value, *args, **kwargs): + tracker.cache_writes.append({"key": key, "method": "async_set_cache"}) + return await original_async_set(key, value, *args, **kwargs) + + cache.async_get_cache = tracked_async_get + cache.async_set_cache = tracked_async_set + return cache + + +def _create_valid_token( + api_key: str, + team_id: str, + user_id: str, + has_team_member_spend: bool = False, + org_id: Optional[str] = None, +) -> UserAPIKeyAuth: + """Create a UserAPIKeyAuth with team_id and user_id set.""" + hashed = hash_token(api_key) + return UserAPIKeyAuth( + token=hashed, + api_key=api_key, + team_id=team_id, + user_id=user_id, + org_id=org_id, + models=["gpt-4", "gpt-3.5-turbo"], + max_budget=100.0, + spend=10.0, + team_spend=50.0, + team_max_budget=1000.0, + team_models=["gpt-4", "gpt-3.5-turbo"], + team_member_spend=5.0 if has_team_member_spend else None, + last_refreshed_at=time.time(), + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + +def _create_team_object(team_id: str) -> LiteLLM_TeamTableCachedObj: + """Create a team table object for caching.""" + return LiteLLM_TeamTableCachedObj( + team_id=team_id, + models=["gpt-4", "gpt-3.5-turbo"], + max_budget=1000.0, + spend=50.0, + tpm_limit=10000, + rpm_limit=100, + last_refreshed_at=time.time(), + ) + + +def _create_user_object(user_id: str) -> LiteLLM_UserTable: + """Create a user table object for caching.""" + return LiteLLM_UserTable( + user_id=user_id, + max_budget=500.0, + spend=25.0, + models=["gpt-4"], + tpm_limit=5000, + rpm_limit=50, + user_role=LitellmUserRoles.INTERNAL_USER, + user_email="test@example.com", + ) + + +# ============================================================================ +# TEST: get_key_object cache behavior +# ============================================================================ + + +@pytest.mark.asyncio +async def test_get_key_object_warm_cache(): + """ + Test get_key_object with a warm cache - should hit cache, no DB query. + """ + api_key = "sk-test-key-warm" + team_id = "team-123" + user_id = "user-456" + hashed_token = hash_token(api_key) + + valid_token = _create_valid_token(api_key, team_id, user_id) + + # Create cache with pre-populated data + cache = DualCache(in_memory_cache=InMemoryCache()) + await cache.async_set_cache(key=hashed_token, value=valid_token) + + # Track cache operations + tracker = CacheCallTracker() + tracked_cache = _wrap_cache_with_tracker(cache, tracker) + + # Mock prisma client (should NOT be called for warm cache) + mock_prisma = MagicMock() + mock_prisma.get_data = AsyncMock() + + result = await get_key_object( + hashed_token=hashed_token, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + ) + + summary = tracker.get_summary() + + # Should have exactly 1 cache read + assert summary["total_cache_reads"] == 1 + assert hashed_token in summary["cache_read_keys"] + + # Prisma should NOT have been called + mock_prisma.get_data.assert_not_called() + + # Result should be the cached token + assert result.token == hashed_token + + +@pytest.mark.asyncio +async def test_get_key_object_cold_cache(): + """ + Test get_key_object with a cold cache - should miss cache, query DB. + """ + api_key = "sk-test-key-cold" + team_id = "team-123" + user_id = "user-456" + hashed_token = hash_token(api_key) + + valid_token = _create_valid_token(api_key, team_id, user_id) + + # Create empty cache + cache = DualCache(in_memory_cache=InMemoryCache()) + + tracker = CacheCallTracker() + tracked_cache = _wrap_cache_with_tracker(cache, tracker) + + # Mock prisma client to return token on DB query + mock_prisma = MagicMock() + mock_prisma.get_data = AsyncMock(return_value=valid_token) + + await get_key_object( + hashed_token=hashed_token, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + ) + + summary = tracker.get_summary() + + # Should have 1 cache read (miss) and at least 1 cache write (populate cache) + assert summary["total_cache_reads"] >= 1 + + # Prisma SHOULD have been called + mock_prisma.get_data.assert_called_once() + + +# ============================================================================ +# TEST: get_team_object cache behavior +# ============================================================================ + + +@pytest.mark.asyncio +async def test_get_team_object_warm_cache(): + """ + Test get_team_object with a warm cache - should hit cache, no DB query. + """ + team_id = "team-warm-123" + team_obj = _create_team_object(team_id) + + cache = DualCache(in_memory_cache=InMemoryCache()) + cache_key = f"team_id:{team_id}" + await cache.async_set_cache(key=cache_key, value=team_obj) + + tracker = CacheCallTracker() + tracked_cache = _wrap_cache_with_tracker(cache, tracker) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_teamtable = MagicMock() + mock_prisma.db.litellm_teamtable.find_unique = AsyncMock() + + await get_team_object( + team_id=team_id, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + ) + + summary = tracker.get_summary() + + assert summary["total_cache_reads"] >= 1 + assert cache_key in summary["cache_read_keys"] + + # DB should NOT have been called + mock_prisma.db.litellm_teamtable.find_unique.assert_not_called() + + +# ============================================================================ +# TEST: get_user_object cache behavior +# ============================================================================ + + +@pytest.mark.asyncio +async def test_get_user_object_warm_cache(): + """ + Test get_user_object with a warm cache - should hit cache, no DB query. + """ + user_id = "user-warm-456" + user_obj = _create_user_object(user_id) + + cache = DualCache(in_memory_cache=InMemoryCache()) + await cache.async_set_cache(key=user_id, value=user_obj) + + tracker = CacheCallTracker() + tracked_cache = _wrap_cache_with_tracker(cache, tracker) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_usertable = MagicMock() + mock_prisma.db.litellm_usertable.find_unique = AsyncMock() + + await get_user_object( + user_id=user_id, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + user_id_upsert=False, + ) + + summary = tracker.get_summary() + + assert summary["total_cache_reads"] >= 1 + assert user_id in summary["cache_read_keys"] + + # DB should NOT have been called + mock_prisma.db.litellm_usertable.find_unique.assert_not_called() + + +# ============================================================================ +# TEST: get_team_membership cache behavior +# ============================================================================ + + +@pytest.mark.asyncio +async def test_get_team_membership_warm_cache(): + """ + Test get_team_membership with a warm cache - should hit cache, no DB query. + """ + user_id = "user-tm-456" + team_id = "team-tm-123" + + membership_dict = { + "user_id": user_id, + "team_id": team_id, + "spend": 3.0, + "budget_id": None, + "litellm_budget_table": None, + } + + cache = DualCache(in_memory_cache=InMemoryCache()) + # Cache key format used by get_team_membership + cache_key = f"team_membership:{user_id}:{team_id}" + await cache.async_set_cache(key=cache_key, value=membership_dict) + + tracker = CacheCallTracker() + tracked_cache = _wrap_cache_with_tracker(cache, tracker) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_teammembership = MagicMock() + mock_prisma.db.litellm_teammembership.find_unique = AsyncMock() + + await get_team_membership( + user_id=user_id, + team_id=team_id, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + ) + + summary = tracker.get_summary() + + assert summary["total_cache_reads"] >= 1 + assert cache_key in summary["cache_read_keys"] + + # DB should NOT have been called + mock_prisma.db.litellm_teammembership.find_unique.assert_not_called() + + +# ============================================================================ +# TEST: Document duplicate team membership cache key issue +# ============================================================================ + + +@pytest.mark.asyncio +async def test_team_membership_cache_key_duplication(): + """ + Document the team membership duplicate cache key issue: + + Team membership is queried via TWO different cache keys: + 1. "{team_id}_{user_id}" - used in user_api_key_auth.py:1048 + 2. "team_membership:{user_id}:{team_id}" - used in auth_checks.py:960 (get_team_membership) + + This test documents that both keys refer to the same data but use different + cache key formats, potentially leading to duplicate lookups. + """ + user_id = "user-dup-456" + team_id = "team-dup-123" + + # The two different cache keys used for the same data + key_format_1 = f"{team_id}_{user_id}" # user_api_key_auth format + key_format_2 = f"team_membership:{user_id}:{team_id}" # auth_checks format + + _ = { + "user_id": user_id, + "team_id": team_id, + "spend": 3.0, + } + + # Document that these are different keys + assert ( + key_format_1 != key_format_2 + ), "Cache keys should be different (this is the bug)" + + # Document that these are different keys + assert ( + key_format_1 != key_format_2 + ), "Cache keys should be different (this is the bug)" + + +# ============================================================================ +# TEST: Full hot path network count summary +# ============================================================================ + + +@pytest.mark.asyncio +async def test_full_hot_path_network_count(): + """ + Summary test that counts all network operations when processing + a request with a key that has team_id and user_id attached. + + This test verifies the baseline number of cache operations expected + on a fully warm cache path. + """ + api_key = "sk-test-full-path" + team_id = "team-full-123" + user_id = "user-full-456" + hashed_token = hash_token(api_key) + + # Create all objects + valid_token = _create_valid_token( + api_key, team_id, user_id, has_team_member_spend=True + ) + team_obj = _create_team_object(team_id) + user_obj = _create_user_object(user_id) + membership_data = LiteLLM_TeamMembership( + user_id=user_id, + team_id=team_id, + spend=3.0, + budget_id=None, + litellm_budget_table=None, + ) + + # Pre-populate cache with all data + cache = DualCache(in_memory_cache=InMemoryCache()) + await cache.async_set_cache(key=hashed_token, value=valid_token) + await cache.async_set_cache(key=f"team_id:{team_id}", value=team_obj) + await cache.async_set_cache(key=user_id, value=user_obj) + await cache.async_set_cache( + key=f"team_membership:{user_id}:{team_id}", value=membership_data.model_dump() + ) + await cache.async_set_cache( + key=f"{team_id}_{user_id}", value=membership_data.model_dump() + ) + + # Create tracker AFTER populating cache + tracker = CacheCallTracker() + tracked_cache = _wrap_cache_with_tracker(cache, tracker) + + # Mock prisma (should not be called on warm cache) + mock_prisma = MagicMock() + + # Call each function to simulate the hot path + await get_key_object( + hashed_token=hashed_token, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + ) + + await get_team_object( + team_id=team_id, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + ) + + await get_user_object( + user_id=user_id, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + user_id_upsert=False, + ) + + await get_team_membership( + user_id=user_id, + team_id=team_id, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + ) + + summary = tracker.get_summary() + + # Assertions for expected baseline + # On warm cache: 4 reads (key, team, user, team_membership) + assert ( + summary["total_cache_reads"] == 4 + ), f"Expected 4 cache reads on warm path, got {summary['total_cache_reads']}" + + # No DB queries on warm cache + assert ( + summary["total_db_queries"] == 0 + ), f"Expected 0 DB queries on warm path, got {summary['total_db_queries']}" + + # Total network requests should be exactly 4 on warm cache + assert ( + summary["total_network_requests"] == 4 + ), f"Expected 4 total network requests on warm path, got {summary['total_network_requests']}" # ============================================================================ @@ -540,7 +540,7 @@ async def test_get_user_object_missing_user_negative_cache(): mock_prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=None) for _ in range(3): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="User doesn't exist in db\\."): await get_user_object( user_id=user_id, prisma_client=mock_prisma, @@ -570,7 +570,7 @@ async def test_get_user_object_missing_user_rechecks_after_expiry(): mock_prisma.db.litellm_usertable = MagicMock() mock_prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=None) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="User doesn't exist in db\\."): await get_user_object( user_id=user_id, prisma_client=mock_prisma, @@ -586,7 +586,7 @@ async def test_get_user_object_missing_user_rechecks_after_expiry(): time.time() - (db_cache_expiry + 1), ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="User doesn't exist in db\\."): await get_user_object( user_id=user_id, prisma_client=mock_prisma, diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index ecf7f89d487..9301176f3ed 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -1588,7 +1588,7 @@ class TestCheckCompleteCredentialsBlocksSSRF: "litellm.proxy.auth.auth_utils.validate_url", side_effect=SSRFError(f"blocked: {blocked_url}"), ): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='is rejected by the SSRF guard') as exc_info: check_complete_credentials( { "model": "gpt-4", @@ -2144,7 +2144,7 @@ class TestIsRequestBodySafeBlocksEndpointTargetingFields: ], ) def test_endpoint_targeting_field_in_request_body_is_rejected(self, field): - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request') as exc: is_request_body_safe( request_body={"model": "gpt-4", field: "https://attacker.example"}, general_settings={}, @@ -2165,7 +2165,7 @@ class TestIsRequestBodySafeBlocksEndpointTargetingFields: # on the blocklist into an SSRF / credential-exfil hole. Verify # that supplying an api_key (alongside the banned param) does NOT # bypass the gate — it can only be opened by an admin opt-in. - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request') as exc: is_request_body_safe( request_body={ "model": "gpt-4", @@ -2722,7 +2722,7 @@ class TestObservabilityCallbackBans: ], ) def test_observability_field_in_request_body_root_is_rejected(self, field): - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request') as exc: is_request_body_safe( request_body={"model": "gpt-4", field: "attacker-value"}, general_settings={}, @@ -2752,7 +2752,7 @@ class TestObservabilityCallbackBans: # Verifies the metadata walk: a value smuggled inside ``metadata`` # or ``litellm_metadata`` is just as dangerous as the same field # at the body root, and must hit the same gate. - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request') as exc: is_request_body_safe( request_body={ "model": "gpt-4", @@ -2787,7 +2787,7 @@ class TestObservabilityCallbackBans: ) def test_observability_field_in_litellm_params_metadata_is_rejected(self): - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request: turn_off_message_logging is not allowed') as exc: is_request_body_safe( request_body={ "model": "gpt-4", @@ -2814,7 +2814,7 @@ class TestObservabilityCallbackBans: # the ``isinstance(dict)`` guard. import json - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request: langfuse_host is not allowed in request') as exc: is_request_body_safe( request_body={ "model": "gpt-4", @@ -2887,7 +2887,7 @@ def test_model_level_allow_does_not_skip_subsequent_banned_params(monkeypatch): lambda model, param, request_body_value, llm_router: param == "api_base", ) - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request: langfuse_host is not allowed in request') as exc: is_request_body_safe( request_body={ "model": "gpt-4", @@ -2958,7 +2958,7 @@ class TestPricingInjectionBlocked: ], ) def test_pricing_field_rejected_by_default(self, field, value): - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request') as exc: is_request_body_safe( request_body={"model": "gpt-4", field: value}, general_settings={}, diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index a9e12beb54b..99a0a4c0a8b 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -2589,7 +2589,7 @@ async def test_find_and_validate_raises_when_required_team_not_found(): # Token without team info jwt_token = {"sub": "user-1"} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="No team found in token\\. Checked team_id field 'None' and") as exc_info: await JWTAuthManager.find_and_validate_specific_team_id( jwt_handler=jwt_handler, jwt_valid_token=jwt_token, @@ -2916,7 +2916,7 @@ async def test_find_and_validate_specific_team_id_hints_bracket_notation(): # token has roles as a list — dot-notation won't find anything token = {"roles": ["team1"]} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="is not supported\\. Use 'roles' instead — LiteLLM") as exc_info: await JWTAuthManager.find_and_validate_specific_team_id( jwt_handler=handler, jwt_valid_token=token, @@ -2947,7 +2947,7 @@ async def test_find_and_validate_specific_team_id_hints_bracket_index_notation() handler = _make_jwt_handler("roles[0]") token = {"roles": ["team1"]} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="is not supported in team_id_jwt_field\\. Use 'roles' instead") as exc_info: await JWTAuthManager.find_and_validate_specific_team_id( jwt_handler=handler, jwt_valid_token=token, @@ -2977,7 +2977,7 @@ async def test_find_and_validate_specific_team_id_no_hint_for_valid_field(): handler = _make_jwt_handler("appid") token = {} # no appid — triggers the "no team found" path - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="No team found in token\\. Checked team_id field 'appid' and") as exc_info: await JWTAuthManager.find_and_validate_specific_team_id( jwt_handler=handler, jwt_valid_token=token, @@ -4807,7 +4807,7 @@ async def test_multi_issuer_jwt_unknown_issuer_falls_back_to_global_jwks(monkeyp kid="issuer-key", ) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='Missing JWT Public Key URL from environment\\.') as exc: await jwt_handler.auth_jwt(token=token) assert "Missing JWT Public Key URL from environment." in str(exc.value) @@ -4838,7 +4838,7 @@ async def test_multi_issuer_jwt_rejects_wrong_audience(monkeypatch): kid="issuer-key", ) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match="Validation fails: Audience doesn't match") as exc: await jwt_handler.auth_jwt(token=token) assert "Validation fails" in str(exc.value) @@ -4881,7 +4881,7 @@ async def test_multi_issuer_jwt_same_kid_does_not_cross_issuer_keys(monkeypatch) kid=shared_kid, ) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='Validation fails: Signature verification failed') as exc: await jwt_handler.auth_jwt(token=token) assert "Validation fails" in str(exc.value) @@ -4936,7 +4936,7 @@ def test_multi_issuer_jwt_requires_audience_unless_explicitly_disabled( issuer = "https://issuer.example.com" jwks_url = f"{issuer}/keys" - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='must configure audience or set') as exc: LiteLLM_JWTAuth( issuers=[ { @@ -4953,7 +4953,7 @@ def test_multi_issuer_jwt_rejects_audience_with_disable_audience_validation(): issuer = "https://issuer.example.com" jwks_url = f"{issuer}/keys" - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='cannot set audience and disable_audience_validation=True') as exc: LiteLLM_JWTAuth( issuers=[ { diff --git a/tests/test_litellm/proxy/auth/test_oauth2_proxy_hook.py b/tests/test_litellm/proxy/auth/test_oauth2_proxy_hook.py index dcbfd281e01..2d81d48de1e 100644 --- a/tests/test_litellm/proxy/auth/test_oauth2_proxy_hook.py +++ b/tests/test_litellm/proxy/auth/test_oauth2_proxy_hook.py @@ -141,7 +141,7 @@ async def test_refuses_to_map_non_identity_fields(configure_proxy, privileged_fi configure_proxy(mappings={privileged_field: f"x-{privileged_field}"}) request = _request_with_headers({f"x-{privileged_field}": "proxy_admin"}) - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='proxy auth refuses to map non-identity UserAPIKeyAuth') as exc: await handle_oauth2_proxy_request(request) assert privileged_field in str(exc.value) diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 6d6e20e9c36..636c5480d67 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -42,7 +42,7 @@ def test_non_admin_config_update_route_rejected(): request.query_params = {} # Test that calling /config/update route raises HTTPException with 403 status - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -134,7 +134,7 @@ def test_user_banner_update_rejected_for_non_admin(): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -1814,7 +1814,7 @@ def test_internal_user_blocked_from_global_spend_routes(route): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -1843,7 +1843,7 @@ def test_internal_user_view_only_blocked_from_global_spend_routes(route): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, @@ -2046,7 +2046,7 @@ def test_internal_user_blocked_from_admin_viewer_logs_routes(route): if route not in INTERNAL_USER_BLOCKED_SUBSET: return - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -2530,7 +2530,7 @@ def test_non_admin_non_team_admin_cannot_access_config_update_but_can_attempt_re ) # /config/update is still blocked - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -3188,7 +3188,7 @@ def test_internal_user_blocked_from_search_tool_writes(route): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index ab7e3d9701c..043bbb5b76a 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -5337,7 +5337,7 @@ async def test_random_non_sk_token_is_rejected(monkeypatch): patch("litellm.proxy.proxy_server.master_key", "sk-master"), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='LiteLLM Virtual Key expected\\.') as exc_info: await user_api_key_auth( request=mock_request, api_key="Bearer not-a-real-token", @@ -5539,7 +5539,7 @@ async def test_real_jwt_still_requires_license_when_jwt_auth_enabled(monkeypatch patch("litellm.proxy.proxy_server.master_key", "sk-master"), patch("litellm.proxy.proxy_server.prisma_client", None), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='JWT Auth is an enterprise only feature\\. You must be a') as exc_info: await user_api_key_auth( request=mock_request, api_key=f"Bearer {jwt_token}", diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index eb40f54a1f3..be29269fe25 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -84,7 +84,7 @@ class TestPollingErrorSurfacing: } with patch("requests.get", return_value=mock_response) as mock_get, patch("time.sleep"): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Your litellm CLI is out of date and uses a login flow') as exc_info: _poll_for_ready_data("http://test/sso/cli/poll/sk-legacy") assert mock_get.call_count == 1 @@ -151,7 +151,7 @@ class TestStartCliSsoFlowErrors: mock_response.status_code = 404 with patch("requests.post", return_value=mock_response): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Either --base-url is wrong, or the proxy is older than') as exc_info: _start_cli_sso_flow("https://old-proxy.example.com") message = str(exc_info.value) @@ -167,7 +167,7 @@ class TestStartCliSsoFlowErrors: mock_response.json.return_value = {"detail": "Too many CLI login attempts. Try again later."} with patch("requests.post", return_value=mock_response): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Too many CLI login attempts\\. Try again later\\.') as exc_info: _start_cli_sso_flow("https://test.example.com") assert "HTTP 429" in str(exc_info.value) @@ -183,7 +183,7 @@ class TestStartCliSsoFlowErrors: mock_response.text = "Sign in to corporate VPN" with patch("requests.post", return_value=mock_response): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='A proxy, load balancer, or auth gateway in front of') as exc_info: _start_cli_sso_flow("https://test.example.com") message = str(exc_info.value) @@ -197,7 +197,7 @@ class TestStartCliSsoFlowErrors: from litellm.proxy.client.cli.commands.auth import _start_cli_sso_flow with patch("requests.post", side_effect=requests.ConnectionError("Connection refused")): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Connection refused\\. Check that the proxy is running') as exc_info: _start_cli_sso_flow("https://unreachable.example.com") message = str(exc_info.value) diff --git a/tests/test_litellm/proxy/client/cli/test_pkce_login.py b/tests/test_litellm/proxy/client/cli/test_pkce_login.py index 70f481d5cfa..f58bd0ff412 100644 --- a/tests/test_litellm/proxy/client/cli/test_pkce_login.py +++ b/tests/test_litellm/proxy/client/cli/test_pkce_login.py @@ -622,7 +622,7 @@ def test_fresh_api_key_never_hands_out_a_rotated_key_it_could_not_save(): def save(_record): raise OSError("disk full") - with pytest.raises(OSError): + with pytest.raises(OSError, match="disk full"): _fresh(STORED, save, http, now=lambda: 999_950.0) diff --git a/tests/test_litellm/proxy/client/test_models.py b/tests/test_litellm/proxy/client/test_models.py index b2485032a37..33f963b74af 100644 --- a/tests/test_litellm/proxy/client/test_models.py +++ b/tests/test_litellm/proxy/client/test_models.py @@ -472,14 +472,14 @@ def test_get_invalid_params(): client = ModelsManagementClient(base_url="http://localhost:8000") # Test with no parameters - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Exactly one of model_id or model_name must be provided') as exc_info: client.get() assert "Exactly one of model_id or model_name must be provided" in str( exc_info.value ) # Test with both parameters - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Exactly one of model_id or model_name must be provided') as exc_info: client.get(model_id="123", model_name="gpt-4") assert "Exactly one of model_id or model_name must be provided" in str( exc_info.value diff --git a/tests/test_litellm/proxy/common_utils/test_callback_utils.py b/tests/test_litellm/proxy/common_utils/test_callback_utils.py index 515a7b27c7b..77ada4c11a9 100644 --- a/tests/test_litellm/proxy/common_utils/test_callback_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_callback_utils.py @@ -586,7 +586,7 @@ def test_initialize_callbacks_on_proxy_rejects_class_valued_entry(probe_config_p silently never run the hook. Config load must fail instead.""" entry = f"{_PROBE_MODULE_NAME}.FloorMaxTokens" - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='litellm_settings\\.callbacks entry') as exc_info: _load_callbacks([entry], probe_config_path) message = str(exc_info.value) @@ -609,7 +609,7 @@ def test_initialize_callbacks_on_proxy_rejects_non_dispatchable_values( ): entry = f"{_PROBE_MODULE_NAME}.{attribute}" - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='litellm_settings\\.callbacks entry') as exc_info: _load_callbacks([entry], probe_config_path) message = str(exc_info.value) @@ -621,7 +621,7 @@ def test_initialize_callbacks_on_proxy_rejects_non_dispatchable_values( def test_initialize_callbacks_on_proxy_rejects_class_valued_non_list_value(probe_config_path): entry = f"{_PROBE_MODULE_NAME}.FloorMaxTokens" - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='litellm_settings\\.callbacks entry') as exc_info: _load_callbacks(entry, probe_config_path) assert entry in str(exc_info.value) diff --git a/tests/test_litellm/proxy/common_utils/test_path_utils.py b/tests/test_litellm/proxy/common_utils/test_path_utils.py index c8d58fa8259..8936d910777 100644 --- a/tests/test_litellm/proxy/common_utils/test_path_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_path_utils.py @@ -42,5 +42,5 @@ class TestSafeFilename: safe_filename("..") def test_empty_rejected(self): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Empty or unsafe filename'): safe_filename("") diff --git a/tests/test_litellm/proxy/common_utils/test_timezone_utils.py b/tests/test_litellm/proxy/common_utils/test_timezone_utils.py index 7f686c53c95..dc3917cb48e 100644 --- a/tests/test_litellm/proxy/common_utils/test_timezone_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_timezone_utils.py @@ -130,16 +130,16 @@ def test_parse_budget_reset_time_unset_defaults_to_midnight(): def test_parse_budget_reset_time_invalid_string_raises(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="hour 'HH:MM' or 'HH:MM:SS' string, e\\.g\\."): parse_budget_reset_time("25:00") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Invalid budget_reset_time 'noon'; expected a"): parse_budget_reset_time("noon") def test_parse_budget_reset_time_non_string_raises(): # Unquoted "12:00" in YAML parses to the int 720; it must fail loudly, # not silently fall back to midnight. - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="hour 'HH:MM' string, e\\.g\\."): parse_budget_reset_time(720) diff --git a/tests/test_litellm/proxy/conftest.py b/tests/test_litellm/proxy/conftest.py index 61752997f0f..65e12b7d777 100644 --- a/tests/test_litellm/proxy/conftest.py +++ b/tests/test_litellm/proxy/conftest.py @@ -18,6 +18,7 @@ from prisma.errors import ClientNotConnectedError _PROXY_MODULE_GLOBALS_TO_ISOLATE = ( "master_key", "prisma_client", + "llm_router", ) @@ -56,7 +57,10 @@ def pytest_runtest_setup(item): Without this, a leaked value (e.g. master_key set by a sibling test) flips the auth short-circuit in user_api_key_auth and causes unrelated - tests in the same xdist worker to return 401 instead of 200. + tests in the same xdist worker to return 401 instead of 200. A leaked + llm_router does the same to anything that reads the running router out + of sys.modules, such as the PTU rollup's deployment scan, which then + counts a sibling test's deployments as if the proxy owned them. This must be a hook pair, not an autouse fixture: an autouse fixture in the root conftest requests monkeypatch, so monkeypatch's undo stack diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_logs_partition_manager.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_logs_partition_manager.py index e949afce57b..4ea655b8871 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_logs_partition_manager.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_logs_partition_manager.py @@ -308,9 +308,9 @@ async def test_partition_maintenance_issues_nothing_when_the_budget_is_already_s def test_unsupported_interval_raises(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Unsupported partition interval: year'): period_start(date(2026, 6, 1), "year") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Unsupported partition interval: year'): next_period_start(date(2026, 6, 1), "year") diff --git a/tests/test_litellm/proxy/db/test_prisma_planned_engine_restart.py b/tests/test_litellm/proxy/db/test_prisma_planned_engine_restart.py index c8e0338eeaa..95e794012ec 100644 --- a/tests/test_litellm/proxy/db/test_prisma_planned_engine_restart.py +++ b/tests/test_litellm/proxy/db/test_prisma_planned_engine_restart.py @@ -913,7 +913,7 @@ async def test_health_check_alerts_for_non_connection_errors_during_a_replacemen await _yield_to_loop() assert wrapper._reconnection_lock.locked() is True - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='malformed SELECT'): await client.health_check() gate.set() diff --git a/tests/test_litellm/proxy/db/test_spend_log_tool_index.py b/tests/test_litellm/proxy/db/test_spend_log_tool_index.py index 71073fd216e..9c4fbbf41aa 100644 --- a/tests/test_litellm/proxy/db/test_spend_log_tool_index.py +++ b/tests/test_litellm/proxy/db/test_spend_log_tool_index.py @@ -327,7 +327,7 @@ class TestFlushToolUsageTransactions: async def test_non_connection_errors_do_not_retry(self): prisma = MagicMock() prisma.db.batch_ = MagicMock(side_effect=ValueError("bad data")) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="bad data"): await flush_tool_usage_transactions(prisma_client=prisma, transactions=[_transaction("r1")]) prisma.db.batch_.assert_called_once() diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py index 3adf8b8407d..ceb59571389 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py @@ -56,7 +56,7 @@ def _patched(guardrail: BedrockGuardrail, http_response): def test_init_rejects_both_identifier_and_checks(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Bedrock guardrail accepts either'): BedrockGuardrail(guardrailIdentifier="gid", checks=CONTENT_FILTER_CHECKS) @@ -304,7 +304,7 @@ async def test_truncated_pii_ignored_when_pii_check_not_configured(): @pytest.mark.asyncio async def test_checks_with_guardrail_version_rejected(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Bedrock guardrail accepts either'): BedrockGuardrail(checks=CONTENT_FILTER_CHECKS, guardrailVersion="DRAFT") diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_enkryptai.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_enkryptai.py index e6c94a4c3cd..d31f462a185 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_enkryptai.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_enkryptai.py @@ -178,7 +178,7 @@ class TestEnkryptAIGuardrailHooks: with patch.object( enkryptai_guardrail.async_handler, "post", return_value=mock_response ): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='violation\\(s\\) detected') as exc_info: await enkryptai_guardrail.async_pre_call_hook( user_api_key_dict=mock_user_api_key_dict, cache=MagicMock(), diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py index 5be0d43c250..523ec1a37b4 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py @@ -767,7 +767,7 @@ class TestErrorHandling: "API Error", request=MagicMock(), response=MagicMock(status_code=500) ), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Generic Guardrail API failed: API Error') as exc_info: await generic_guardrail.apply_guardrail( inputs={"texts": ["test"]}, request_data=mock_request_data_input, @@ -786,7 +786,7 @@ class TestErrorHandling: "post", side_effect=httpx.RequestError("Connection failed", request=MagicMock()), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Generic Guardrail API failed: Connection failed') as exc_info: await generic_guardrail.apply_guardrail( inputs={"texts": ["test"]}, request_data=mock_request_data_input, @@ -810,7 +810,7 @@ class TestErrorHandling: "post", side_effect=httpx.RequestError("Connection failed", request=MagicMock()), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Generic Guardrail API failed: Connection failed') as exc_info: await guardrail.apply_guardrail( inputs={"texts": ["test"]}, request_data=mock_request_data_input, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py index 89b6af27719..14c0d2f9435 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py @@ -2334,7 +2334,7 @@ async def test_async_moderation_hook_api_error_fail_on_error_true(): } # Should raise the exception since fail_on_error is True - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="API Error") as exc_info: await guardrail.async_moderation_hook( data=request_data, user_api_key_dict=mock_user_api_key_dict, @@ -2374,7 +2374,7 @@ async def test_async_moderation_hook_api_error_fail_on_error_false(): # Even with fail_on_error=False, the decorator may still raise the exception # This test verifies that the exception is properly logged and handled - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="API Error") as exc_info: await guardrail.async_moderation_hook( data=request_data, user_api_key_dict=mock_user_api_key_dict, @@ -2865,7 +2865,7 @@ async def test_skip_unscannable_still_fails_closed_on_api_error(): "post", AsyncMock(side_effect=Exception("model armor upstream 500")), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='model armor upstream') as exc_info: await guardrail.async_pre_call_hook( user_api_key_dict=UserAPIKeyAuth(), cache=MagicMock(spec=DualCache), diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py index 2284f2b678a..8f29ba66814 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py @@ -5652,7 +5652,7 @@ class TestPanwAirsTimeoutCoercion: assert isinstance(params.timeout, float) def test_litellm_params_rejects_garbage_timeout(self): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='validation error for LitellmParams'): LitellmParams( guardrail="panw_prisma_airs", mode="pre_call", diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py index a3d86034f70..81604e22c87 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py @@ -90,12 +90,12 @@ def test_config_model_wiring(): def test_init_rejects_empty_api_key(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='api_key must be non-empty'): StraikerGuardrail(api_key="") def test_init_rejects_invalid_fallback(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="unreachable_fallback must be 'fail_open' or 'fail_closed';"): StraikerGuardrail(api_key="k", unreachable_fallback="nope") @@ -109,7 +109,7 @@ def test_supported_hooks_limited_to_pre_and_post(): def test_during_call_mode_rejected_at_init(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Event hook GuardrailEventHooks\\.during_call is not in the'): StraikerGuardrail(api_key="k", event_hook="during_call") diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py index 4b381b67f0e..0c5addbc143 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py @@ -124,7 +124,7 @@ class TestToolPermissionGuardrail: assert rule_id is None def test_rule_requires_name_or_type(self): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='validation error for ToolPermissionRule'): ToolPermissionGuardrail( guardrail_name="invalid-rule", rules=[{"id": "no_target", "decision": "allow"}], @@ -1042,7 +1042,7 @@ class TestToolPermissionGuardrailInMemoryUpdate: assert guardrail._check_tool_permission("Secret")[0] is False assert guardrail._check_tool_permission("Other")[0] is True - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Invalid regex for tool_name in rule 'bad': unterminated"): guardrail.update_in_memory_litellm_params( LitellmParams( guardrail="tool_permission", diff --git a/tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py b/tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py index 45dec4ddb2d..bd2553b3280 100644 --- a/tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py +++ b/tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py @@ -230,7 +230,7 @@ def test_parse_judge_verdict_reraises_when_no_json(): def test_parse_judge_verdict_rejects_json_non_object(): """Valid JSON that is not an object (e.g. a bare list) raises ValueError.""" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='judge response is not a JSON object'): _parse_judge_verdict("[1, 2, 3]") diff --git a/tests/test_litellm/proxy/hooks/test_sensitive_data_routing.py b/tests/test_litellm/proxy/hooks/test_sensitive_data_routing.py index 78d2c3af0f3..35c0f8deaf1 100644 --- a/tests/test_litellm/proxy/hooks/test_sensitive_data_routing.py +++ b/tests/test_litellm/proxy/hooks/test_sensitive_data_routing.py @@ -227,7 +227,7 @@ class TestCustomGuardrailSensitiveDataRouting: request_data = {"model": "gpt-4"} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Cannot route sensitive data without a session_id\\. Ensure') as exc_info: guardrail.raise_sensitive_data_route_exception( route_to_model="on-premise-model", request_data=request_data, diff --git a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py index 8d03857c917..2839acab6b0 100644 --- a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py +++ b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py @@ -150,7 +150,7 @@ async def test_no_leak_on_over_limit_rejection(rate_limiter): f"estimated={estimated}, limit={user_api_key_dict.tpm_limit}" ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Limit type: tokens\\. Current limit') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -685,7 +685,7 @@ async def test_contentless_request_reserves_minimum(rate_limiter): f"counter should be 2, got {counter_after_two}" ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Limit type: tokens\\. Current limit') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -1319,7 +1319,7 @@ async def test_project_otpm_rejects_multiple_completion_candidates(rate_limiter) "n": 10, } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -1347,7 +1347,7 @@ async def test_project_otpm_reserves_largest_conflicting_output_cap(rate_limiter "max_completion_tokens": 100, } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -1377,7 +1377,7 @@ async def test_project_otpm_rejects_google_genai_native_output_cap( project_metadata={"model_otpm_limit": {model: 50}}, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -1411,7 +1411,7 @@ async def test_project_otpm_rejects_google_genai_native_candidate_count( project_metadata={"model_otpm_limit": {model: 150}}, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -1500,7 +1500,7 @@ async def test_project_otpm_over_limit_rolls_back_itpm_reservation(rate_limiter) "max_tokens": 500, # blows past the 10-token OTPM limit } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -2003,7 +2003,7 @@ async def test_otpm_rejection_does_not_double_refund_combined_tpm(rate_limiter): rate_limit_type="tokens", ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -2060,7 +2060,7 @@ async def test_project_itpm_rejects_pretokenized_embedding_input( "input": embedding_input, } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_itpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -2250,7 +2250,7 @@ async def test_itpm_reservation_accounts_for_audio_content_not_just_text(rate_li ], } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_itpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -2361,7 +2361,7 @@ async def test_itpm_rejects_large_audio_payload_that_would_pass_flat_estimate( ], } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_itpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -2622,7 +2622,7 @@ async def test_explicit_zero_output_responses_call_reserves_effective_provider_m }, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -2850,7 +2850,7 @@ async def test_otpm_rejection_releases_stashed_parallel_slot(rate_limiter): "rate_limit": {"tokens_per_unit": 5, "window_size": 60}, } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler._reserve_project_io_tokens_or_raise( descriptors=[otpm_descriptor], data=data, @@ -3296,7 +3296,7 @@ async def test_rerank_query_and_documents_enforce_project_itpm( project_metadata={"model_itpm_limit": {"rerank-model": 100}}, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_itpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index d4e9ccdca5e..069cfa01178 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -10179,7 +10179,7 @@ async def test_update_key_creator_reassigned_key_blocked(monkeypatch): mock_request = MagicMock() mock_request.query_params = {} - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='User can only create keys for themselves\\. Got') as exc: await update_key_fn( request=mock_request, data=UpdateKeyRequest(key=test_hashed_token, key_alias="hijacked"), diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index 84dee5b05c5..01c0760bd27 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -2325,7 +2325,7 @@ class TestTemporaryMCPSessionEndpoints: "litellm.proxy.management_endpoints.mcp_management_endpoints.validate_and_normalize_mcp_server_payload", MagicMock(), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='User does not have permission to create temporary mcp') as exc_info: await add_session_mcp_server( payload=payload, user_api_key_dict=non_admin, diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 7e4596d154b..42e96ad8659 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -140,7 +140,7 @@ class TestModelManagementAuthChecks: @pytest.mark.asyncio async def test_can_user_make_team_model_call_non_premium_fails(self): """Test that non-premium users cannot make team model calls""" - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='You must be a LiteLLM Enterprise user to use this feature\\.') as exc_info: ModelManagementAuthChecks.can_user_make_team_model_call( team_id="test_team", user_api_key_dict=self.admin_user, @@ -195,7 +195,7 @@ class TestModelManagementAuthChecks: ) prisma_client = MockPrismaClient(team_exists=True) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='You must be a LiteLLM Enterprise user to use this feature\\.') as exc_info: await ModelManagementAuthChecks.allow_team_model_action( model_params=model_params, user_api_key_dict=self.admin_user, @@ -216,7 +216,7 @@ class TestModelManagementAuthChecks: ) prisma_client = MockPrismaClient(team_exists=False) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Team id=nonexistent_team does not exist in db'\\}") as exc_info: await ModelManagementAuthChecks.allow_team_model_action( model_params=model_params, user_api_key_dict=self.admin_user, @@ -257,7 +257,7 @@ class TestModelManagementAuthChecks: ) prisma_client = MockPrismaClient(team_exists=True, user_admin=False) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Team ID=test_team does not match the API key's team") as exc_info: await ModelManagementAuthChecks.can_user_make_model_call( model_params=model_params, user_api_key_dict=self.normal_user, @@ -1483,7 +1483,7 @@ class TestTeamModelUpdate: "litellm.proxy.proxy_server.premium_user", True, ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="does not match the API key's team ID=None, OR you are") as exc_info: await _update_team_model_in_db( db_model=db_model, patch_data=patch_data, @@ -3256,7 +3256,7 @@ class TestPatchModelBlockedAuthGate: new=AsyncMock(return_value=None), ), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Only proxy admins can change a model's blocked flag\\.") as exc_info: await patch_model( model_id="m1", patch_data=updateDeployment(blocked=True), diff --git a/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py b/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py index 92a34b5ee7c..a1c38d26b9d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py @@ -58,7 +58,7 @@ def test_model_info_accepts_valid_ptu_fields(): def test_model_info_rejects_non_positive_count(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='value_error, input_value'): ModelInfo( id="x", team_id="t", @@ -69,7 +69,7 @@ def test_model_info_rejects_non_positive_count(): def test_model_info_rejects_negative_rate(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='value_error, input_value'): ModelInfo( id="x", team_id="t", @@ -82,7 +82,7 @@ def test_model_info_rejects_negative_rate(): def test_model_info_rejects_a_count_beyond_the_cap(): """flat cost multiplies the count by a float, and an unbounded int overflows that conversion, which aborted the rollup for every team rather than skipping one model.""" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='validation error for ModelInfo'): ModelInfo(id="x", team_id="t", ptu_count=10**400, cost_per_ptu_per_hour=2.0) @@ -95,12 +95,12 @@ def test_model_info_accepts_a_count_at_the_cap(): def test_model_info_rejects_a_non_finite_rate(rate): """NaN compares False against every bound, so a bare `< 0` check let it through and the deployment then accrued a flat cost of nan.""" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='value_error, input_value'): ModelInfo(id="x", team_id="t", ptu_count=5, cost_per_ptu_per_hour=rate) def test_model_info_rejects_a_rate_beyond_the_cap(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='validation error for ModelInfo'): ModelInfo(id="x", team_id="t", ptu_count=5, cost_per_ptu_per_hour=ModelInfo.MAX_COST_PER_PTU_PER_HOUR * 2) @@ -148,7 +148,7 @@ def test_validate_helper_passes_full_config(): def test_model_info_rejects_effective_to_before_from(): import datetime - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='validation error for ModelInfo'): ModelInfo( id="x", team_id="t", @@ -186,7 +186,7 @@ def test_model_info_compares_mixed_naive_and_aware_timestamps(): ) assert info.ptu_effective_to is not None - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='validation error for ModelInfo'): ModelInfo( id="x", team_id="t", @@ -698,7 +698,7 @@ class TestAddNewModelPtuGate: with ExitStack() as stack: for active_patch in patches: stack.enter_context(active_patch) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='PTU cost attribution is disabled, so ptu_count') as exc: await add_new_model(model_params=self._ptu_deployment("ptu-gate-model"), user_api_key_dict=admin) assert PTU_COST_ATTRIBUTION_ENV_VAR in str(exc.value) @@ -1273,7 +1273,7 @@ class TestPtuDeploymentsAreNotBilledPerToken: with ExitStack() as stack: for active_patch in patches: stack.enter_context(active_patch) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='A PTU deployment bills by reserved capacity, so') as exc: await add_new_model(model_params=deployment, user_api_key_dict=admin) assert "input_cost_per_token" in str(exc.value) diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 0767288d0bc..e39b09ae073 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -10253,6 +10253,65 @@ async def test_team_info_returns_model_aliases(): assert litellm_model_table.model_aliases == {"gpt-4o": "gpt-4o-team-1"} +@pytest.mark.asyncio +async def test_team_info_hydrates_member_emails_from_the_user_table(): + """/team/info must fill in emails missing from the members_with_roles snapshot. + + members_with_roles is written at add-time, so a member added by user_id alone + carries user_email=None forever. Without this join the Admin UI's member table + shows "-" for a user that has an email on their user row. A stored email is left + exactly as-is. + """ + from fastapi import Request + + from litellm.proxy.management_endpoints import team_endpoints + + team_row = LiteLLM_TeamTable( + team_id="team-1", + members_with_roles=[ + Member(user_id="no-email-on-roster", role="admin"), + Member(user_id="already-stored", user_email="stored@example.com", role="user"), + ], + ) + + mock_prisma = MagicMock() + mock_prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=team_row) + mock_prisma.get_data = AsyncMock(return_value=[]) + + find_many = AsyncMock( + return_value=[ + LiteLLM_UserTable( + user_id="no-email-on-roster", + user_email="real@example.com", + max_budget=None, + spend=0.0, + models=[], + ) + ] + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch.object(team_endpoints, "get_all_team_memberships", AsyncMock(return_value=[])), + patch.object(team_endpoints, "UserRepository") as repo, + ): + repo.return_value.table.find_many = find_many + + response = await team_endpoints.team_info( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + members = response["team_info"].members_with_roles + assert [(m.user_id, m.user_email) for m in members] == [ + ("no-email-on-roster", "real@example.com"), + ("already-stored", "stored@example.com"), + ] + # only the member actually missing an email is looked up + assert find_many.await_args.kwargs["where"] == {"user_id": {"in": ["no-email-on-roster"]}} + + @pytest.mark.asyncio async def test_update_model_table_clears_aliases_with_empty_map(): """``model_aliases={}`` on /team/update must persist an empty map (json.dumps({})) @@ -11369,6 +11428,141 @@ async def test_resolve_existing_member_user_ids_skips_the_query_when_no_user_ids repo.return_value.table.find_many.assert_not_awaited() +def _user_row(user_id: str, user_email: str | None) -> LiteLLM_UserTable: + return LiteLLM_UserTable( + user_id=user_id, user_email=user_email, max_budget=None, spend=0.0, models=[] + ) + + +@pytest.mark.asyncio +async def test_hydrate_member_emails_fills_in_emails_the_roster_snapshot_never_captured(): + """A member added by user_id alone has user_email=None on the stored roster entry. + + /team/info has to fill it in from the user row, or the UI renders "-" for a user + that plainly has an email. + """ + from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_emails + + find_many = AsyncMock(return_value=[_user_row("by-id", "found@example.com")]) + + with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: + repo.return_value.table.find_many = find_many + + hydrated = await _hydrate_member_emails( + prisma_client=MagicMock(), + members=[Member(user_id="by-id", role="admin")], + ) + + assert [(m.user_id, m.user_email, m.role) for m in hydrated] == [("by-id", "found@example.com", "admin")] + find_many.assert_awaited_once() + assert find_many.await_args.kwargs["where"] == {"user_id": {"in": ["by-id"]}} + + +@pytest.mark.asyncio +async def test_hydrate_member_emails_never_overwrites_a_stored_email(): + """The snapshot wins wherever it has a value - hydration only fills blanks. + + Overwriting would be a real behavior change to /team/info; filling a null is not. + """ + from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_emails + + find_many = AsyncMock(return_value=[_user_row("has-email", "current@example.com")]) + + with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: + repo.return_value.table.find_many = find_many + + hydrated = await _hydrate_member_emails( + prisma_client=MagicMock(), + members=[Member(user_id="has-email", user_email="stored@example.com", role="user")], + ) + + assert hydrated[0].user_email == "stored@example.com" + # nothing was missing, so no round-trip either + find_many.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_hydrate_member_emails_leaves_members_alone_when_the_user_row_has_no_email(): + """A user row with no email leaves the member as-is rather than inventing one.""" + from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_emails + + with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: + repo.return_value.table.find_many = AsyncMock(return_value=[_user_row("no-email", None)]) + + hydrated = await _hydrate_member_emails( + prisma_client=MagicMock(), + members=[Member(user_id="no-email", role="user"), Member(user_email="e@example.com", role="user")], + ) + + assert [m.user_email for m in hydrated] == [None, "e@example.com"] + + +@pytest.mark.asyncio +async def test_hydrate_member_emails_skips_the_query_when_every_member_has_one(): + """No blanks means /team/info pays for no extra query.""" + from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_emails + + with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: + repo.return_value.table.find_many = AsyncMock() + + hydrated = await _hydrate_member_emails( + prisma_client=MagicMock(), + members=[Member(user_id="a", user_email="a@example.com", role="user")], + ) + + assert hydrated[0].user_email == "a@example.com" + repo.return_value.table.find_many.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_update_team_members_list_stamps_email_for_a_member_added_by_user_id(): + """Identity resolution runs both ways, so new roster entries stop being born blank. + + Previously only user_id was backfilled (from email); a member added by user_id + was written with user_email=None forever. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + _update_team_members_list, + ) + + mock_team = MagicMock(spec=LiteLLM_TeamTable) + mock_team.members_with_roles = [] + + await _update_team_members_list( + data=TeamMemberAddRequest(team_id="test-team-123", member=Member(user_id="new-user-123", role="user")), + complete_team_data=mock_team, + updated_users=[_user_row("new-user-123", "new@example.com")], + ) + + assert len(mock_team.members_with_roles) == 1 + assert mock_team.members_with_roles[0].user_email == "new@example.com" + + +@pytest.mark.asyncio +async def test_update_team_members_list_stamps_email_for_each_member_in_a_bulk_add(): + """Same both-ways resolution for the list branch.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _update_team_members_list, + ) + + mock_team = MagicMock(spec=LiteLLM_TeamTable) + mock_team.members_with_roles = [] + + await _update_team_members_list( + data=TeamMemberAddRequest( + team_id="test-team-123", + member=[Member(user_id="u1", role="user"), Member(user_email="u2@example.com", role="admin")], + ), + complete_team_data=mock_team, + updated_users=[_user_row("u1", "u1@example.com"), _user_row("u2", "u2@example.com")], + ) + + assert [(m.user_id, m.user_email) for m in mock_team.members_with_roles] == [ + ("u1", "u1@example.com"), + ("u2", "u2@example.com"), + ] + + def test_pre_existing_user_ids_counts_ids_filled_in_by_member_resolution(): """An id the member-resolution step filled in came from a matched row, so it pre-existed. diff --git a/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py b/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py index e8a74e41dae..3a32b3cc128 100644 --- a/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py +++ b/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py @@ -458,7 +458,7 @@ class TestUsageAiChatServiceAccountGuard: _resolve_fetch_kwargs, ) - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Non-admin caller has user_id=None; refusing to issue an') as exc_info: _resolve_fetch_kwargs( fn_name="get_usage_data", fn_args={"start_date": "2025-01-01", "end_date": "2025-01-31"}, diff --git a/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py b/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py index e4b031ade57..1acb8e7e016 100644 --- a/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py +++ b/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py @@ -671,7 +671,7 @@ async def test_non_callable_validator_is_rejected_with_clean_500(): def test_parse_schema_duplicate_error_lists_offending_keys(): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='team_metadata_schema contains duplicate keys: app_name') as exc_info: parse_team_metadata_schema( [{"key": "cost_center"}, {"key": "app_name"}, {"key": "cost_center"}, {"key": "app_name"}] ) diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_batch_guardrails.py b/tests/test_litellm/proxy/openai_files_endpoint/test_batch_guardrails.py index a05b8ae530c..6ce7af1e2ee 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_batch_guardrails.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_batch_guardrails.py @@ -781,7 +781,7 @@ async def test_the_rewrite_closes_its_own_output_when_it_cannot_finish(): original_read = bg._read_spooled bg._read_spooled = _boom try: - with pytest.raises(OSError): + with pytest.raises(OSError, match='no space left on device'): rewrite_batch_input_file(source, result) finally: bg.tempfile.SpooledTemporaryFile = real diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index f994fba371b..d3237f5f49d 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -2714,7 +2714,7 @@ class TestMilvusProxyRoute: None ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Vector store not found for missing-store') as exc_info: await milvus_proxy_route( endpoint="vectors/search", request=mock_request, @@ -2779,7 +2779,7 @@ class TestMilvusProxyRoute: mock_vector_store ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='api_base not found in vector store configuration for') as exc_info: await milvus_proxy_route( endpoint="vectors/search", request=mock_request, @@ -2988,7 +2988,7 @@ class TestOpenAIPassthroughRoute: "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials", return_value=None, ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Required 'OPENAI_API_KEY' in environment to make") as exc_info: await openai_proxy_route( endpoint="v1/chat/completions", request=mock_request, @@ -3177,7 +3177,7 @@ class TestCursorProxyRoute: [], ), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Cursor API key not found\\. Add Cursor credentials via') as exc_info: await cursor_proxy_route( endpoint="v0/agents", request=mock_request, diff --git a/tests/test_litellm/proxy/policy_engine/test_policy_versioning.py b/tests/test_litellm/proxy/policy_engine/test_policy_versioning.py index ebebfde5cd3..b6633779326 100644 --- a/tests/test_litellm/proxy/policy_engine/test_policy_versioning.py +++ b/tests/test_litellm/proxy/policy_engine/test_policy_versioning.py @@ -157,7 +157,7 @@ class TestUpdatePolicyDraftOnly: prod_row = _make_row(policy_id="pid-1", version_status="production") prisma.db.litellm_policytable.find_unique = AsyncMock(return_value=prod_row) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Error updating policy in DB: Only draft versions can be') as exc_info: await registry.update_policy_in_db( policy_id="pid-1", policy_request=PolicyUpdateRequest(description="new"), @@ -341,7 +341,7 @@ class TestUpdateVersionStatus: draft = _make_row(policy_id="d-1", version_status="draft") prisma.db.litellm_policytable.find_unique = AsyncMock(return_value=draft) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Error updating version status: Cannot promote draft') as exc_info: await registry.update_version_status( policy_id="d-1", new_status="production", diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 54ae279d005..47f01fe096d 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -1504,7 +1504,7 @@ async def test_ProxyConfig__init_non_llm_configs_premium_invalid_worker_registry async def test_ProxyConfig__init_non_llm_configs_worker_registry_requires_premium(monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) pc = ProxyConfig() - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Trying to use `worker_registry`You must be a LiteLLM') as exc_info: await pc._init_non_llm_configs( config={ "worker_registry": [ @@ -1769,7 +1769,7 @@ def test_ProxyConfig_initialize_secret_manager_none_noop(): def test_ProxyConfig_initialize_secret_manager_invalid_kms_raises(): pc = ProxyConfig() - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Invalid Key Management System selected'): pc.initialize_secret_manager(key_management_system="not-a-real-kms") diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 15a3e6609f0..b2ec500d045 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -3263,7 +3263,7 @@ async def test_provider_budget_over(disable_budget_sync): model_list=MODEL_LIST, ) - with pytest.raises(Exception) as e: + with pytest.raises(Exception, match='No deployments available - crossed budget: Exceeded budget') as e: await router.acompletion( model="azure-gpt-4o", messages=[{"role": "user", "content": "Hello, world!"}], @@ -5096,7 +5096,7 @@ def test_resolve_spend_report_scope_missing_caller_value_400(): @pytest.mark.parametrize("bad_column", ["metadata", "end_user", "evil; DROP TABLE", ""]) def test_scoped_spend_report_sql_rejects_unknown_column(bad_column): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Unsupported spend report scope column'): spend_management_endpoints._scoped_spend_report_sql(scope_column=bad_column) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 9c97b2683b2..e9a22a75607 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -9,7 +9,6 @@ from typing import Any, Final, cast from typing_extensions import ReadOnly, TypedDict import pytest -from fastapi.testclient import TestClient sys.path.insert( 0, os.path.abspath("../../../..") @@ -31,8 +30,8 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( _get_response_for_spend_logs_payload, _get_spend_logs_metadata, _get_vector_store_request_for_spend_logs_payload, - _hash_api_key_for_spend_log, _is_master_key, + _redact_logged_api_key, _redact_prompt_leaks_in_error_string, _sanitize_error_information_for_spend_logs, _sanitize_guardrail_information_for_spend_logs, @@ -41,6 +40,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( get_logging_payload, get_spend_logs_id, ) +from litellm.proxy.utils import hash_token from litellm.types.utils import ( StandardLoggingHiddenParams, StandardLoggingMetadata, @@ -890,8 +890,6 @@ def test_get_logging_payload_api_key_preserved_when_standard_logging_payload_is_ assert payload["model"] == "openai/gpt-4.1" assert payload["user"] == "test_user" - print(f"✅ Test passed! api_key preserved: {payload['api_key']}") - @pytest.mark.asyncio @patch("litellm.proxy.proxy_server.master_key", "sk-master-key") @@ -1039,18 +1037,6 @@ async def test_api_key_preserved_through_failure_hook_to_database(): assert payload.get("model") == "gpt-3.5-turbo" assert payload.get("user") == "test_user" - print("\n" + "=" * 80) - print("✅ CRITICAL E2E TEST PASSED") - print("=" * 80) - print(f"Token: {data['token']}") - print(f"Payload api_key: {payload_api_key}") - print(f"Match: {data['token'] == payload_api_key}") - print("=" * 80) - print("Production incident bug is FIXED and protected:") - print("- Failed requests preserve api_key through entire flow") - print("- Both SpendLogs AND DailyUserSpend will have correct api_key") - print("=" * 80 + "\n") - @patch("litellm.proxy.proxy_server.master_key", None) @patch("litellm.proxy.proxy_server.general_settings", {}) @@ -2593,6 +2579,219 @@ def test_sanitize_error_information_redacts_pydantic_assignment_form( assert REDACTED_BY_LITELM_STRING in sanitized["error_message"] +# ── _redact_logged_api_key unit tests ────────────────────────────────────── + + +def test_redact_logged_api_key_none_returns_none(): + assert _redact_logged_api_key(None) is None + + +def test_redact_logged_api_key_empty_string_returns_none(): + assert _redact_logged_api_key("") is None + + +def test_redact_logged_api_key_sk_key_is_hashed(): + raw = "sk-1234secret" + result = _redact_logged_api_key(raw) + assert result == hash_token(raw) + assert result is not None + assert not result.startswith("sk-") + assert len(result) == 64 + + +def test_redact_logged_api_key_bearer_sk_equals_sk_hash(): + raw = "sk-1234secret" + result_plain = _redact_logged_api_key(raw) + result_bearer = _redact_logged_api_key(f"Bearer {raw}") + assert result_bearer == result_plain + + +def test_redact_logged_api_key_bearer_case_insensitive(): + raw = "sk-1234secret" + result_lower = _redact_logged_api_key(f"bearer {raw}") + result_upper = _redact_logged_api_key(f"BEARER {raw}") + expected = hash_token(raw) + assert result_lower == expected + assert result_upper == expected + + +def test_redact_logged_api_key_non_sk_raw_key_is_hashed(): + raw = "anthropic-raw-key-xyz" + result = _redact_logged_api_key(raw) + assert result is not None + assert result != raw + assert len(result) == 64 + assert result == hash_token(raw) + + +def test_redact_logged_api_key_already_valid_sha256_passes_through_with_flag(): + already_hashed = hash_token("sk-some-key") + assert len(already_hashed) == 64 + result = _redact_logged_api_key(already_hashed, already_redacted=True) + assert result == already_hashed + assert hash_token(already_hashed) != result # no double-hash + + +def test_redact_logged_api_key_sha256_without_flag_is_hashed(): + already_hashed = hash_token("sk-some-key") + assert len(already_hashed) == 64 + result = _redact_logged_api_key(already_hashed) + assert result is not None + assert result != already_hashed + assert len(result) == 64 + assert result == hash_token(already_hashed) + + +def test_redact_logged_api_key_long_opaque_token_is_hashed(): + raw = "x1" * 450 + assert len(raw) == 900 + result = _redact_logged_api_key(raw) + assert result is not None + assert result != raw + assert raw not in result + assert len(result) == 64 + assert result == hash_token(raw) + + +def test_redact_logged_api_key_hashed_jwt_passes_through(): + jwt_hash = "hashed-jwt-" + "a" * 64 + result = _redact_logged_api_key(jwt_hash, already_redacted=True) + assert result == jwt_hash + + +def test_redact_logged_api_key_hashed_jwt_shape_without_provenance_is_hashed(): + lookalike = "hashed-jwt-" + "a" * 64 + result = _redact_logged_api_key(lookalike) + assert result == hash_token(lookalike) + assert result != lookalike + + +def test_redact_logged_api_key_hashed_jwt_trailing_newline_is_hashed(): + trailing = "hashed-jwt-" + "a" * 64 + "\n" + result = _redact_logged_api_key(trailing, already_redacted=True) + assert result == hash_token(trailing) + assert result != trailing + + +def test_redact_logged_api_key_hashed_jwt_short_suffix_is_hashed(): + short_jwt = "hashed-jwt-tooshort" + result = _redact_logged_api_key(short_jwt) + assert result is not None + assert result != short_jwt + assert len(result) == 64 + assert result == hash_token(short_jwt) + + +def test_redact_logged_api_key_master_key_alias_passes_through(): + from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS + + result = _redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS, already_redacted=True) + assert result == LITELLM_PROXY_MASTER_KEY_ALIAS + + +def test_redact_logged_api_key_master_key_alias_without_provenance_is_hashed(): + from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS + + result = _redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS) + assert result == hash_token(LITELLM_PROXY_MASTER_KEY_ALIAS) + assert result != LITELLM_PROXY_MASTER_KEY_ALIAS + + +def test_get_spend_logs_metadata_keeps_master_key_alias_readable(): + from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS + + meta = _get_spend_logs_metadata( + { + "user_api_key": LITELLM_PROXY_MASTER_KEY_ALIAS, + "user_api_key_hash": LITELLM_PROXY_MASTER_KEY_ALIAS, + } + ) + assert meta["user_api_key"] == LITELLM_PROXY_MASTER_KEY_ALIAS + + +def test_redact_logged_api_key_bearer_only_returns_none(): + # "bearer " with nothing after stripping is equivalent to no key + assert _redact_logged_api_key("bearer ") is None + assert _redact_logged_api_key("Bearer ") is None + assert _redact_logged_api_key("BEARER ") is None + + +# ── _get_spend_logs_metadata key-hash invariant tests ───────────────────── + + +def test_get_spend_logs_metadata_sk_key_hashed(): + raw = "sk-1234secret" + meta = _get_spend_logs_metadata({"user_api_key": raw}) + assert meta["user_api_key"] == hash_token(raw) + assert meta["user_api_key"] is not None + result = meta["user_api_key"] + assert result is not None + assert not result.startswith("sk-") + assert len(result) == 64 + + +def test_get_spend_logs_metadata_bearer_sk_key_hashed_same_as_plain(): + raw = "sk-1234secret" + meta_plain = _get_spend_logs_metadata({"user_api_key": raw}) + meta_bearer = _get_spend_logs_metadata({"user_api_key": f"Bearer {raw}"}) + assert meta_bearer["user_api_key"] == meta_plain["user_api_key"] + + +def test_get_spend_logs_metadata_non_sk_raw_key_hashed(): + raw = "anthropic-raw-key-xyz" + meta = _get_spend_logs_metadata({"user_api_key": raw}) + result = meta["user_api_key"] + assert result is not None + assert result != raw + assert len(result) == 64 + + +def test_get_spend_logs_metadata_already_hashed_unchanged_with_provenance(): + already_hashed = hash_token("sk-some-key") + meta = _get_spend_logs_metadata( + {"user_api_key": already_hashed, "user_api_key_hash": already_hashed} + ) + assert meta["user_api_key"] == already_hashed + assert hash_token(already_hashed) != meta["user_api_key"] # no double-hash + + +def test_get_spend_logs_metadata_already_hashed_no_provenance_is_rehashed(): + already_hashed = hash_token("sk-some-key") + meta = _get_spend_logs_metadata({"user_api_key": already_hashed}) + assert meta["user_api_key"] != already_hashed + assert meta["user_api_key"] == hash_token(already_hashed) + + +def test_get_spend_logs_metadata_provenance_bypass_requires_hash_match(): + already_hashed = hash_token("sk-some-key") + different_hash = hash_token("sk-other-key") + meta = _get_spend_logs_metadata( + {"user_api_key": already_hashed, "user_api_key_hash": different_hash} + ) + assert meta["user_api_key"] == hash_token(already_hashed) + + +def test_get_spend_logs_metadata_hashed_jwt_unchanged(): + jwt_hash = "hashed-jwt-" + "b" * 64 + meta = _get_spend_logs_metadata({"user_api_key": jwt_hash, "user_api_key_hash": jwt_hash}) + assert meta["user_api_key"] == jwt_hash + + +def test_get_spend_logs_metadata_hashed_jwt_shape_without_provenance_is_hashed(): + lookalike = "hashed-jwt-" + "b" * 64 + meta = _get_spend_logs_metadata({"user_api_key": lookalike}) + assert meta["user_api_key"] == hash_token(lookalike) + assert meta["user_api_key"] != lookalike + + +def test_get_spend_logs_metadata_none_key_is_none(): + meta = _get_spend_logs_metadata({"user_api_key": None}) + assert meta["user_api_key"] is None + + +# ── get_logging_payload key-hash invariant tests ─────────────────────────── + + def test_get_logging_payload_uses_recovered_combined_usage_on_failure(): """A request that fails mid-stream has no usable response_obj usage, but the streaming handler recovers the usage from the chunks already delivered and @@ -2749,44 +2948,107 @@ def test_get_logging_payload_cache_hit_keeps_raw_litellm_call_id(): assert json.loads(payload["metadata"])["litellm_call_id"] != payload["request_id"] -class TestHashApiKeyForSpendLog: +class TestSpendLogKeyRedaction: """Regression: plaintext API keys with Bearer prefix were stored in SpendLogs for failed requests (LIT-4121)""" def test_bearer_prefixed_sk_key_is_hashed(self): raw = "Bearer sk-WLi4iRn4JmbVlTaYw12IOA" - result = _hash_api_key_for_spend_log(raw) + result = _redact_logged_api_key(raw) + assert result is not None assert not result.startswith("Bearer") assert not result.startswith("sk-") assert len(result) == 64 def test_bare_sk_key_is_hashed(self): raw = "sk-WLi4iRn4JmbVlTaYw12IOA" - result = _hash_api_key_for_spend_log(raw) + result = _redact_logged_api_key(raw) + assert result is not None assert not result.startswith("sk-") assert len(result) == 64 def test_bearer_lowercase_is_handled(self): raw = "bearer sk-WLi4iRn4JmbVlTaYw12IOA" - result = _hash_api_key_for_spend_log(raw) + result = _redact_logged_api_key(raw) + assert result is not None assert not result.startswith("bearer") assert not result.startswith("sk-") assert len(result) == 64 def test_already_hashed_key_unchanged(self): hashed = "bcfe8173f5447f10be0e7fb37aaa8b97829d5c9e0498232152f9d123456789ab" - assert _hash_api_key_for_spend_log(hashed) == hashed + assert _redact_logged_api_key(hashed, already_redacted=True) == hashed - def test_bearer_prefixed_non_sk_key_strips_prefix(self): + def test_bearer_prefixed_non_sk_key_is_hashed(self): raw = "Bearer some-other-token-format" - result = _hash_api_key_for_spend_log(raw) - assert result == "some-other-token-format" + result = _redact_logged_api_key(raw) + assert result == hash_token("some-other-token-format") + assert result is not None assert not result.startswith("Bearer") def test_bearer_and_bare_produce_same_hash(self): bare = "sk-WLi4iRn4JmbVlTaYw12IOA" bearer = "Bearer sk-WLi4iRn4JmbVlTaYw12IOA" - assert _hash_api_key_for_spend_log(bare) == _hash_api_key_for_spend_log(bearer) + assert _redact_logged_api_key(bare) == _redact_logged_api_key(bearer) + + +@patch("litellm.proxy.proxy_server.master_key", None) +@patch("litellm.proxy.proxy_server.general_settings", {}) +def test_get_logging_payload_non_sk_raw_key_both_fields_hashed(): + raw = "anthropic-raw-key-xyz" + kwargs = { + "model": "openai/gpt-4.1", + "messages": [{"role": "user", "content": "Hello"}], + "call_type": "acompletion", + "litellm_params": { + "metadata": { + "user_api_key": raw, + "user_api_key_user_id": "test_user", + "user_api_key_team_id": "test_team", + } + }, + } + payload = get_logging_payload( + kwargs=kwargs, + response_obj=Exception("error"), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert payload["api_key"] != raw + assert len(payload["api_key"]) == 64 + + parsed_meta = json.loads(payload["metadata"]) + assert parsed_meta["user_api_key"] != raw + assert parsed_meta["user_api_key"] is not None + assert len(parsed_meta["user_api_key"]) == 64 + + +def test_get_logging_payload_keeps_master_key_alias_readable(): + from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS + + kwargs = { + "model": "openai/gpt-4.1", + "messages": [{"role": "user", "content": "Hello"}], + "call_type": "acompletion", + "litellm_params": { + "metadata": { + "user_api_key": LITELLM_PROXY_MASTER_KEY_ALIAS, + "user_api_key_hash": LITELLM_PROXY_MASTER_KEY_ALIAS, + "user_api_key_user_id": "test_user", + } + }, + } + payload = get_logging_payload( + kwargs=kwargs, + response_obj=Exception("error"), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert payload["api_key"] == LITELLM_PROXY_MASTER_KEY_ALIAS + parsed_meta = json.loads(payload["metadata"]) + assert parsed_meta["user_api_key"] == LITELLM_PROXY_MASTER_KEY_ALIAS @patch("litellm.proxy.proxy_server.master_key", None) @@ -3291,3 +3553,69 @@ def test_get_logging_payload_falls_back_to_kwargs_model_when_slp_model_missing() end_time=datetime.datetime.now(timezone.utc), ) assert payload["model"] == "azure_ai/model_router/model-router" + + +@patch("litellm.proxy.proxy_server.master_key", None) +@patch("litellm.proxy.proxy_server.general_settings", {}) +def test_get_logging_payload_empty_key_slp_none_is_empty_string_not_none_literal(): + kwargs = { + "model": "openai/gpt-4.1", + "messages": [{"role": "user", "content": "Hello"}], + "call_type": "acompletion", + "litellm_params": { + "metadata": { + "user_api_key_user_id": "test_user", + } + }, + } + payload = get_logging_payload( + kwargs=kwargs, + response_obj=Exception("error"), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert payload["api_key"] == "", ( + f"Expected empty string but got {payload['api_key']!r}; " + "dropping _redact_logged_api_key's 'or \"\"' guard would yield 'None' here" + ) + + +def test_get_spend_logs_metadata_sibling_fields_preserved(): + raw = "anthropic-raw-key-xyz" + meta = _get_spend_logs_metadata( + { + "user_api_key": raw, + "user_api_key_alias": "my-alias", + "user_api_key_team_id": "team-123", + } + ) + assert meta["user_api_key"] == hash_token(raw) + assert meta["user_api_key_alias"] == "my-alias" + assert meta["user_api_key_team_id"] == "team-123" + + +def test_redact_logged_api_key_partial_sha256_is_hashed(): + partial_hex = "a" * 63 + result = _redact_logged_api_key(partial_hex) + assert result is not None + assert result != partial_hex + assert len(result) == 64 + assert result == hash_token(partial_hex) + + +def test_redact_logged_api_key_bearer_already_hashed_passes_through_with_flag(): + already_hashed = hash_token("sk-some-key") + assert len(already_hashed) == 64 + result = _redact_logged_api_key(f"Bearer {already_hashed}", already_redacted=True) + assert result == already_hashed + assert hash_token(already_hashed) != result + + +def test_redact_logged_api_key_bearer_sha256_without_flag_is_hashed(): + already_hashed = hash_token("sk-some-key") + assert len(already_hashed) == 64 + result = _redact_logged_api_key(f"Bearer {already_hashed}") + assert result is not None + assert result != already_hashed + assert result == hash_token(already_hashed) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index c693f5ab2cb..716fba370df 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -435,7 +435,7 @@ class TestProxyBaseLLMRequestProcessing: # Test with invalid header value (should raise ValueError when converting to float) headers_with_invalid = {"x-litellm-stream-timeout": "invalid"} - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="could not convert string to float: 'invalid"): LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers_with_invalid) @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/test_enforce_user_param.py b/tests/test_litellm/proxy/test_enforce_user_param.py index 6891123e70e..1001372aeb5 100644 --- a/tests/test_litellm/proxy/test_enforce_user_param.py +++ b/tests/test_litellm/proxy/test_enforce_user_param.py @@ -56,7 +56,7 @@ class TestEnforceUserParamPostGetFiltering: new_callable=AsyncMock, return_value=True, ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="user' param not passed in\\. 'enforce_user_param'=True") as exc_info: await common_checks( request_body=request_body, team_object=None, @@ -175,7 +175,7 @@ class TestEnforceUserParamPostGetFiltering: new_callable=AsyncMock, return_value=True, ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="user' param not passed in\\. 'enforce_user_param'=True") as exc_info: await common_checks( request_body=request_body, team_object=None, @@ -405,7 +405,7 @@ class TestEnforceUserParamEdgeCases: new_callable=AsyncMock, return_value=True, ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="user' param not passed in\\. 'enforce_user_param'=True") as exc_info: await common_checks( request_body=request_body, team_object=None, diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index b1071150f3b..636974d5deb 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -2279,12 +2279,12 @@ def test_get_num_retries_from_request(): # Test case 7: Header present with invalid value (should raise ValueError when int() is called) headers_with_invalid = {"x-litellm-num-retries": "invalid"} - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='invalid literal for int\\(\\) with base'): LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_invalid) # Test case 8: Header present with float string (should raise ValueError when int() is called) headers_with_float = {"x-litellm-num-retries": "3.5"} - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='invalid literal for int\\(\\) with base'): LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_float) # Test case 9: Header present with negative number @@ -2324,7 +2324,7 @@ def test_get_keepalive_seconds_from_request(): # Header present with invalid value raises ValueError, matching the other # x-litellm-* numeric header helpers (_get_timeout_from_request, etc.) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="could not convert string to float: 'not-a-number"): LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request( {"x-litellm-keepalive-seconds": "not-a-number"} ) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 75aa716bb85..83e9095c8ec 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -1507,7 +1507,7 @@ def test_team_info_masking(): "langfuse_public_key": "public-test-key", } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="secr\\*\\*\\*\\*\\*\\*\\*-key', 'langfuse_public_key':") as exc_info: proxy_config._get_team_config( team_id="test_dev", all_teams_config=[team1_info], diff --git a/tests/test_litellm/proxy/test_spend_log_cleanup.py b/tests/test_litellm/proxy/test_spend_log_cleanup.py index ce0b6b755cc..bf1538183ab 100644 --- a/tests/test_litellm/proxy/test_spend_log_cleanup.py +++ b/tests/test_litellm/proxy/test_spend_log_cleanup.py @@ -82,10 +82,10 @@ def test_spend_log_cleanup_cron_scheduling(): assert trigger_weekly is not None # Invalid cron expression should raise ValueError - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Wrong number of fields; got'): CronTrigger.from_crontab("invalid cron") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='is higher than the maximum value'): CronTrigger.from_crontab("60 25 * * *") # Invalid minute and hour diff --git a/tests/test_litellm/proxy/test_team_org_move.py b/tests/test_litellm/proxy/test_team_org_move.py index 2dc961bec85..064e9de550e 100644 --- a/tests/test_litellm/proxy/test_team_org_move.py +++ b/tests/test_litellm/proxy/test_team_org_move.py @@ -97,7 +97,7 @@ class TestValidateTeamOrgChange: team = _make_team(member_ids=["sso-user-001"]) org = _make_org(members=[]) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Cannot move team to organization\\. Team has user_id') as exc_info: validate_team_org_change( team=team, organization=org, llm_router=router, is_proxy_admin=False ) diff --git a/tests/test_litellm/proxy/utils/helpers/test_team_configs.py b/tests/test_litellm/proxy/utils/helpers/test_team_configs.py index 0e0906892b0..185d4d26ff4 100644 --- a/tests/test_litellm/proxy/utils/helpers/test_team_configs.py +++ b/tests/test_litellm/proxy/utils/helpers/test_team_configs.py @@ -66,7 +66,7 @@ def test_is_valid_team_configs_short_circuits_when_team_id_none(): def test_is_valid_team_configs_raises_on_model_not_in_team_models(): team_config = {"models": ["gpt-4o"]} request_data = {"model": "claude-haiku"} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='claude-haiku\\. Valid models for team are') as exc_info: _is_valid_team_configs( team_id="team-1", team_config=team_config, diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py index 7057a112c83..93c99c7fd04 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py @@ -561,7 +561,7 @@ async def test_update_spend_logs_does_not_requeue_non_transport_failures( proxy_logging.failure_handler = AsyncMock() mock_prisma_client.spend_log_transactions = [] - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="bad payload"): await ProxyUpdateSpend.update_spend_logs( n_retry_times=1, prisma_client=mock_prisma_client, diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_callback_capabilities_class.py b/tests/test_litellm/proxy/utils/proxy_logging/test_callback_capabilities_class.py index 9452e8042bd..75a91177f00 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_callback_capabilities_class.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_callback_capabilities_class.py @@ -181,7 +181,7 @@ def test_has_streaming_callbacks_error_when_resolution_fails(monkeypatch): "get_custom_logger_compatible_class", lambda *a, **kw: (_ for _ in ()).throw(ValueError("nope")), ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="nope"): ProxyLogging.has_streaming_callbacks() diff --git a/tests/test_litellm/responses/test_responses_websocket_all_providers.py b/tests/test_litellm/responses/test_responses_websocket_all_providers.py index 2d523bfdeb3..e8333214ea8 100644 --- a/tests/test_litellm/responses/test_responses_websocket_all_providers.py +++ b/tests/test_litellm/responses/test_responses_websocket_all_providers.py @@ -84,7 +84,7 @@ class TestResponsesAPIWebSocketSupport: def test_azure_websocket_url_requires_api_base(self): config = AzureOpenAIResponsesAPIConfig() - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='api_base is required for Azure WebSocket'): config.get_websocket_url(api_base=None, litellm_params={}) def test_azure_model_not_in_websocket_url(self): diff --git a/tests/test_litellm/router_strategy/adaptive_router/test_bandit.py b/tests/test_litellm/router_strategy/adaptive_router/test_bandit.py index ab322f0fb37..78390cc1193 100644 --- a/tests/test_litellm/router_strategy/adaptive_router/test_bandit.py +++ b/tests/test_litellm/router_strategy/adaptive_router/test_bandit.py @@ -100,7 +100,7 @@ def test_score_combines_quality_and_cost(): def test_pick_best_empty_dict_raises(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='pick_best called with no models'): pick_best({}, {}) diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py index 73491490b14..60b1166de73 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py @@ -656,7 +656,7 @@ async def test_negation_all_excluded_raises(): enable_tag_filtering=True, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -699,7 +699,7 @@ async def test_negation_ban_only_cannot_escape_default_pool(): # Sending only "!default" must NOT route to the paid deployment. # The base pool for ban-only is the default pool; banning the only # default deployment should raise rather than falling through to paid. - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -969,7 +969,7 @@ async def test_negation_exhausts_entire_fallback_chain(): enable_tag_filtering=True, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="primary", messages=[{"role": "user", "content": "hi"}], @@ -1719,7 +1719,7 @@ async def test_required_and_unmatched_raises_by_default(): enable_tag_filtering=True, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -1751,7 +1751,7 @@ async def test_required_and_combined_with_positive_unmatched_raises_by_default() enable_tag_filtering=True, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -1973,7 +1973,7 @@ async def test_negation_combined_with_positive_unmatched_raises_by_default(): enable_tag_filtering=True, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -2131,7 +2131,7 @@ async def test_mixed_constraint_survivor_unmatched_by_positive_tag_raises_by_def enable_tag_filtering=True, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -2224,7 +2224,7 @@ async def test_allow_fail_open_denied_when_request_includes_unknown_tag(): enable_tag_filtering=True, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -2538,7 +2538,7 @@ async def test_plain_tag_exhaustion_with_universal_default_tag_raises_by_default "litellm.router._async_get_cooldown_deployments", new=AsyncMock(return_value=["quality-high-1", "quality-high-2"]), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -2767,7 +2767,7 @@ async def test_allow_fail_open_raises_when_inherited_constraint_alone_is_unsatis # allow_fail_open unset. router = _eu_region_router() - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="chat", messages=[{"role": "user", "content": "hi"}], @@ -2941,7 +2941,7 @@ async def test_tagged_request_direct_to_plain_group_still_rejected(): # tag filtering must reject exactly as before. router = _tagged_marker_router() - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gemini-flash", messages=[{"role": "user", "content": "hi"}], @@ -2962,7 +2962,7 @@ async def test_caller_forged_consumption_stamp_is_neutralized_by_the_hook(): # tag filtering runs. router = _tagged_marker_router() - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gemini-flash", messages=[{"role": "user", "content": "hi"}], @@ -2984,7 +2984,7 @@ async def test_inherited_constraint_still_applies_to_the_routed_tier(): # ®ion:eu comes from key/team policy (present in inherited_tags): # consuming the router-selecting "route" tag must not also discard the # inherited requirement, so a tier without the tag still raises... - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await _tagged_marker_router().acompletion( model="gpt4o", messages=[{"role": "user", "content": "hi"}], diff --git a/tests/test_litellm/sandbox/test_e2b_sandbox.py b/tests/test_litellm/sandbox/test_e2b_sandbox.py index cc5b12156a1..e01b9120416 100644 --- a/tests/test_litellm/sandbox/test_e2b_sandbox.py +++ b/tests/test_litellm/sandbox/test_e2b_sandbox.py @@ -293,7 +293,7 @@ async def test_public_lifecycle_create_run_delete(): @pytest.mark.asyncio async def test_unsupported_provider_raises(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="not-a-provider' is not a valid SandboxProviders"): await litellm.acreate_sandbox(provider="not-a-provider") diff --git a/tests/test_litellm/secret_managers/test_base_secret_manager.py b/tests/test_litellm/secret_managers/test_base_secret_manager.py index cba6a99ab7f..e1ccb91c381 100644 --- a/tests/test_litellm/secret_managers/test_base_secret_manager.py +++ b/tests/test_litellm/secret_managers/test_base_secret_manager.py @@ -32,7 +32,7 @@ from litellm.secret_managers.base_secret_manager import raise_if_unsafe_secret_n ], ) def test_raise_if_unsafe_secret_name_rejects_traversal_and_line_breaks(secret_name): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Invalid secret_name'): raise_if_unsafe_secret_name(secret_name) diff --git a/tests/test_litellm/test_github_close_low_quality_prs.py b/tests/test_litellm/test_github_close_low_quality_prs.py index e3b653dde64..2a891ca72f5 100644 --- a/tests/test_litellm/test_github_close_low_quality_prs.py +++ b/tests/test_litellm/test_github_close_low_quality_prs.py @@ -697,7 +697,7 @@ class TestListOpenItemsNoCap: def test_list_open_items_rejects_unknown_kind(self, closer_module): shared = self._shared(closer_module) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="kind must be 'pr' or 'issue', got 'both"): shared.list_open_items("both", repo="o/r", fields="number") def test_fetch_open_prs_delegates_with_no_cap(self, closer_module, monkeypatch): diff --git a/tests/test_litellm/test_github_triage_with_llm.py b/tests/test_litellm/test_github_triage_with_llm.py index 96b77e80457..ddffb978b48 100644 --- a/tests/test_litellm/test_github_triage_with_llm.py +++ b/tests/test_litellm/test_github_triage_with_llm.py @@ -665,11 +665,11 @@ class TestParseVerdict: assert triage_module.parse_verdict(raw)["verdict"] == "pass" def test_should_raise_for_unparseable_text(self, triage_module): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='could not extract JSON from LLM response: not even close to'): triage_module.parse_verdict("not even close to json") def test_should_raise_for_empty(self, triage_module): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='empty LLM response'): triage_module.parse_verdict("") diff --git a/tests/test_litellm/test_redact_string_in_error_paths.py b/tests/test_litellm/test_redact_string_in_error_paths.py index 8eddc2b1a5a..1c4d91397d1 100644 --- a/tests/test_litellm/test_redact_string_in_error_paths.py +++ b/tests/test_litellm/test_redact_string_in_error_paths.py @@ -234,7 +234,7 @@ class TestRouterFallbackFailureTracebackRedaction: raise ValueError(f"primary deployment failed api_key={secret}") except ValueError as original_exception: with caplog.at_level(logging.DEBUG, logger="LiteLLM Router"): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='primary deployment failed api_key=sk-testsecretvalu'): await router.async_function_with_fallbacks_common_utils( e=original_exception, disable_fallbacks=False, diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index 3aa4bc58f13..c645a67ef84 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -131,7 +131,7 @@ def test_get_redis_url_from_environment_missing_host_port(monkeypatch): monkeypatch.delenv("REDIS_PORT", raising=False) # Call the function and expect a ValueError - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match="Either 'REDIS_URL' or both 'REDIS_HOST' and 'REDIS_PORT") as excinfo: get_redis_url_from_environment() # Check the error message @@ -149,7 +149,7 @@ def test_get_redis_url_from_environment_missing_port(monkeypatch): monkeypatch.setenv("REDIS_HOST", "redis-server") # Call the function and expect a ValueError - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match="Either 'REDIS_URL' or both 'REDIS_HOST' and 'REDIS_PORT") as excinfo: get_redis_url_from_environment() # Check the error message diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index b50dc92c220..a47525749f6 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -953,7 +953,7 @@ async def test_arouter_filter_team_based_models(): assert result is not None # FAILS - with pytest.raises(Exception) as e: + with pytest.raises(Exception, match='No deployments available for selected model, Try again in') as e: result = await router.acompletion( model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hello, world!"}], @@ -1225,7 +1225,7 @@ def test_add_invalid_provider_to_router(): ], ) - with pytest.raises(Exception) as e: + with pytest.raises(Exception, match='Unsupported provider - vertex_ai_eu') as e: router.add_deployment( Deployment( model_name="vertex_ai/*", @@ -1320,7 +1320,7 @@ async def test_router_ageneric_api_call_with_fallbacks_helper(): with patch.object(router, "async_get_available_deployment") as mock_get_deployment: mock_get_deployment.side_effect = Exception("No deployment available") - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='No deployment available') as exc_info: await router._ageneric_api_call_with_fallbacks_helper( model="gpt-3.5-turbo", original_generic_function=mock_generic_function, @@ -1394,7 +1394,7 @@ async def test_router_ageneric_api_call_with_fallbacks_helper(): with patch.object( router, "async_routing_strategy_pre_call_checks" ) as mock_pre_call_checks: - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Mock failure') as exc_info: await router._ageneric_api_call_with_fallbacks_helper( model="gpt-3.5-turbo", original_generic_function=mock_failing_function, @@ -3737,7 +3737,7 @@ def test_count_pre_call_check_tokens_across_api_surfaces(): assert string_input_tokens > 0 assert list_input_tokens > 0 - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Either messages or input must be provided to count tokens'): router._count_pre_call_check_tokens(messages=None, input=None) diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 5bb854c12e0..dc210f900bf 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -1706,7 +1706,7 @@ def test_an_incomplete_reservation_is_refused_rather_than_served(dropped): state the operator was trying to leave.""" incomplete = {k: v for k, v in _PTU_MODEL_INFO.items() if k != dropped} - with pytest.raises(ValueError) as raised: + with pytest.raises(ValueError, match="PTU configuration on model 'gpt") as raised: _ptu_router(model_info=incomplete, litellm_params={"input_cost_per_token": 5e-06}) assert "gpt-4o-ptu" in str(raised.value) @@ -1726,7 +1726,7 @@ def test_the_refusal_reason_is_the_one_the_model_endpoint_answers_with(dropped, incomplete = {k: v for k, v in _PTU_MODEL_INFO.items() if k != dropped} assert ptu_config_error(incomplete) == expected - with pytest.raises(ValueError) as raised: + with pytest.raises(ValueError, match="PTU configuration on model 'gpt") as raised: _ptu_router(model_info=incomplete) assert expected in str(raised.value) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 041c60e0ba6..075b455e4b5 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4370,7 +4370,7 @@ class TestVertexEmbeddingEncodingFormat: assert "encoding_format" not in optional_params def test_encoding_format_base64_still_rejected_without_drop_params(self): - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match='To drop these, set `litellm\\.drop_params=True` or for proxy') as excinfo: litellm.utils.get_optional_params_embeddings( model="gemini-embedding-001", encoding_format="base64", diff --git a/tests/test_litellm/types/test_router.py b/tests/test_litellm/types/test_router.py index 5ce5eca4954..accd3b32a0d 100644 --- a/tests/test_litellm/types/test_router.py +++ b/tests/test_litellm/types/test_router.py @@ -87,5 +87,5 @@ def test_pricing_strings_are_coerced_to_float(): def test_invalid_pricing_is_rejected(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='validation error for ModelInfo'): ModelInfo(id="x", input_cost_per_token="free")