From e2932d22c0b6d6e0a01c4346fe304525eed9bf8e Mon Sep 17 00:00:00 2001 From: Nitish Agarwal <1592163+nitishagar@users.noreply.github.com> Date: Thu, 18 Jun 2026 14:06:36 +0530 Subject: [PATCH 01/12] fix(spend-tracking): hash raw api keys before persisting to spend logs (#30670) The Request Logs "Key Hash" column binds to the SpendLogs metadata user_api_key field, which _get_spend_logs_metadata copied verbatim from request metadata without hashing. The top-level api_key column only hashed values starting with sk-. So a non-sk- passthrough provider key, or a Bearer-prefixed key on paths that do not strip it, could be persisted and shown in plaintext. A single helper now redacts both fields at the SpendLogs builder: it strips a Bearer prefix, passes through values already a sha256 hash or hashed-jwt- identifier, and otherwise hashes with hash_token. sk- keys still map to the same canonical hash, so log and spend correlation is unchanged --- .../spend_tracking/spend_tracking_utils.py | 42 ++- .../test_spend_tracking_utils.py | 270 ++++++++++++++++-- 2 files changed, 290 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index aef06a3c668..e8aa2325a18 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -24,6 +24,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, reconstruct_model_name, ) +from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error @@ -65,6 +66,24 @@ def _is_master_key(api_key: Optional[str], _master_key: Optional[str]) -> bool: return secrets.compare_digest(api_key, _master_key) +_HASHED_JWT_RE = re.compile(r"^hashed-jwt-[a-fA-F0-9]{64}$") + + +def _redact_logged_api_key( + value: str | None, *, already_hashed: bool = False +) -> str | None: + if not isinstance(value, str) or not value: + return None + stripped = re.sub(r"(?i)^bearer ", "", value) + if not stripped: + return None + if already_hashed and is_valid_sha256_hash(stripped): + return stripped + if _HASHED_JWT_RE.match(stripped): + return stripped + return hash_token(stripped) + + def _get_spend_logs_metadata( metadata: Optional[dict], applied_guardrails: Optional[List[str]] = None, @@ -121,6 +140,16 @@ def _get_spend_logs_metadata( key: metadata.get(key) for key in SpendLogsMetadata.__annotations__.keys() } ) + _raw_key = clean_metadata.get("user_api_key") + _trusted_hash = metadata.get("user_api_key_hash") if metadata else None + _already_hashed = ( + isinstance(_trusted_hash, str) + and is_valid_sha256_hash(_trusted_hash) + and _trusted_hash == _raw_key + ) + clean_metadata["user_api_key"] = _redact_logged_api_key( + _raw_key, already_hashed=_already_hashed + ) clean_metadata["applied_guardrails"] = applied_guardrails clean_metadata["batch_models"] = batch_models clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata @@ -283,10 +312,13 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs "completion_tokens", 0 ) standard_logging_total_tokens = standard_logging_payload.get("total_tokens", 0) - if api_key is not None and isinstance(api_key, str): - if api_key.startswith("sk-"): - # hash the api_key - api_key = hash_token(api_key) + _trusted_hash = metadata.get("user_api_key_hash") if metadata else None + _key_already_hashed = ( + isinstance(_trusted_hash, str) + and is_valid_sha256_hash(_trusted_hash) + and _trusted_hash == api_key + ) + api_key = _redact_logged_api_key(api_key, already_hashed=_key_already_hashed) or "" if ( standard_logging_payload is not None @@ -299,8 +331,6 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs end_user_id = end_user_id or standard_logging_payload["metadata"].get( "user_api_key_end_user_id" ) - # BUG FIX: Don't overwrite api_key when standard_logging_payload is None - # The api_key was already extracted from metadata (line 243) and hashed (lines 256-259) request_tags = ( safe_dumps(metadata.get("tags", [])) if isinstance(metadata.get("tags", []), list) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 0c7511589de..67d57d2d6bb 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -7,7 +7,6 @@ from datetime import timezone from typing import Any, cast import pytest -from fastapi.testclient import TestClient sys.path.insert( 0, os.path.abspath("../../../..") @@ -15,7 +14,6 @@ sys.path.insert( from unittest.mock import AsyncMock, MagicMock, patch -import litellm from litellm.constants import ( LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, @@ -30,12 +28,14 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( _get_spend_logs_metadata, _get_vector_store_request_for_spend_logs_payload, _is_master_key, + _redact_logged_api_key, _redact_prompt_leaks_in_error_string, _sanitize_error_information_for_spend_logs, _sanitize_request_body_for_spend_logs_payload, _should_store_prompts_and_responses_in_spend_logs, get_logging_payload, ) +from litellm.proxy.utils import hash_token from litellm.types.utils import ( StandardLoggingHiddenParams, StandardLoggingMetadata, @@ -661,8 +661,6 @@ def test_get_logging_payload_api_key_preserved_when_standard_logging_payload_is_ assert payload["model"] == "openai/gpt-4.1" assert payload["user"] == "test_user" - print(f"✅ Test passed! api_key preserved: {payload['api_key']}") - @pytest.mark.asyncio @patch("litellm.proxy.proxy_server.master_key", "sk-master-key") @@ -810,18 +808,6 @@ async def test_api_key_preserved_through_failure_hook_to_database(): assert payload.get("model") == "gpt-3.5-turbo" assert payload.get("user") == "test_user" - print("\n" + "=" * 80) - print("✅ CRITICAL E2E TEST PASSED") - print("=" * 80) - print(f"Token: {data['token']}") - print(f"Payload api_key: {payload_api_key}") - print(f"Match: {data['token'] == payload_api_key}") - print("=" * 80) - print("Production incident bug is FIXED and protected:") - print("- Failed requests preserve api_key through entire flow") - print("- Both SpendLogs AND DailyUserSpend will have correct api_key") - print("=" * 80 + "\n") - @patch("litellm.proxy.proxy_server.master_key", None) @patch("litellm.proxy.proxy_server.general_settings", {}) @@ -2073,3 +2059,255 @@ def test_sanitize_error_information_redacts_pydantic_assignment_form( assert sanitized is not None assert "leaked-via-pydantic-msg" not in sanitized["error_message"] assert REDACTED_BY_LITELM_STRING in sanitized["error_message"] + + +# ── _redact_logged_api_key unit tests ────────────────────────────────────── + + +def test_redact_logged_api_key_none_returns_none(): + assert _redact_logged_api_key(None) is None + + +def test_redact_logged_api_key_empty_string_returns_none(): + assert _redact_logged_api_key("") is None + + +def test_redact_logged_api_key_sk_key_is_hashed(): + raw = "sk-1234secret" + result = _redact_logged_api_key(raw) + assert result == hash_token(raw) + assert result is not None + assert not result.startswith("sk-") + assert len(result) == 64 + + +def test_redact_logged_api_key_bearer_sk_equals_sk_hash(): + raw = "sk-1234secret" + result_plain = _redact_logged_api_key(raw) + result_bearer = _redact_logged_api_key(f"Bearer {raw}") + assert result_bearer == result_plain + + +def test_redact_logged_api_key_bearer_case_insensitive(): + raw = "sk-1234secret" + result_lower = _redact_logged_api_key(f"bearer {raw}") + result_upper = _redact_logged_api_key(f"BEARER {raw}") + expected = hash_token(raw) + assert result_lower == expected + assert result_upper == expected + + +def test_redact_logged_api_key_non_sk_raw_key_is_hashed(): + raw = "anthropic-raw-key-xyz" + result = _redact_logged_api_key(raw) + assert result is not None + assert result != raw + assert len(result) == 64 + assert result == hash_token(raw) + + +def test_redact_logged_api_key_already_valid_sha256_passes_through_with_flag(): + already_hashed = hash_token("sk-some-key") + assert len(already_hashed) == 64 + result = _redact_logged_api_key(already_hashed, already_hashed=True) + assert result == already_hashed + assert hash_token(already_hashed) != result # no double-hash + + +def test_redact_logged_api_key_sha256_without_flag_is_hashed(): + already_hashed = hash_token("sk-some-key") + assert len(already_hashed) == 64 + result = _redact_logged_api_key(already_hashed) + assert result is not None + assert result != already_hashed + assert len(result) == 64 + assert result == hash_token(already_hashed) + + +def test_redact_logged_api_key_hashed_jwt_passes_through(): + jwt_hash = "hashed-jwt-" + "a" * 64 + result = _redact_logged_api_key(jwt_hash) + assert result == jwt_hash + + +def test_redact_logged_api_key_hashed_jwt_short_suffix_is_hashed(): + short_jwt = "hashed-jwt-tooshort" + result = _redact_logged_api_key(short_jwt) + assert result is not None + assert result != short_jwt + assert len(result) == 64 + assert result == hash_token(short_jwt) + + +def test_redact_logged_api_key_bearer_only_returns_none(): + # "bearer " with nothing after stripping is equivalent to no key + assert _redact_logged_api_key("bearer ") is None + assert _redact_logged_api_key("Bearer ") is None + assert _redact_logged_api_key("BEARER ") is None + + +# ── _get_spend_logs_metadata key-hash invariant tests ───────────────────── + + +def test_get_spend_logs_metadata_sk_key_hashed(): + raw = "sk-1234secret" + meta = _get_spend_logs_metadata({"user_api_key": raw}) + assert meta["user_api_key"] == hash_token(raw) + assert meta["user_api_key"] is not None + result = meta["user_api_key"] + assert result is not None + assert not result.startswith("sk-") + assert len(result) == 64 + + +def test_get_spend_logs_metadata_bearer_sk_key_hashed_same_as_plain(): + raw = "sk-1234secret" + meta_plain = _get_spend_logs_metadata({"user_api_key": raw}) + meta_bearer = _get_spend_logs_metadata({"user_api_key": f"Bearer {raw}"}) + assert meta_bearer["user_api_key"] == meta_plain["user_api_key"] + + +def test_get_spend_logs_metadata_non_sk_raw_key_hashed(): + raw = "anthropic-raw-key-xyz" + meta = _get_spend_logs_metadata({"user_api_key": raw}) + result = meta["user_api_key"] + assert result is not None + assert result != raw + assert len(result) == 64 + + +def test_get_spend_logs_metadata_already_hashed_unchanged_with_provenance(): + already_hashed = hash_token("sk-some-key") + meta = _get_spend_logs_metadata( + {"user_api_key": already_hashed, "user_api_key_hash": already_hashed} + ) + assert meta["user_api_key"] == already_hashed + assert hash_token(already_hashed) != meta["user_api_key"] # no double-hash + + +def test_get_spend_logs_metadata_already_hashed_no_provenance_is_rehashed(): + already_hashed = hash_token("sk-some-key") + meta = _get_spend_logs_metadata({"user_api_key": already_hashed}) + assert meta["user_api_key"] != already_hashed + assert meta["user_api_key"] == hash_token(already_hashed) + + +def test_get_spend_logs_metadata_provenance_bypass_requires_hash_match(): + already_hashed = hash_token("sk-some-key") + different_hash = hash_token("sk-other-key") + meta = _get_spend_logs_metadata( + {"user_api_key": already_hashed, "user_api_key_hash": different_hash} + ) + assert meta["user_api_key"] == hash_token(already_hashed) + + +def test_get_spend_logs_metadata_hashed_jwt_unchanged(): + jwt_hash = "hashed-jwt-" + "b" * 64 + meta = _get_spend_logs_metadata({"user_api_key": jwt_hash}) + assert meta["user_api_key"] == jwt_hash + + +def test_get_spend_logs_metadata_none_key_is_none(): + meta = _get_spend_logs_metadata({"user_api_key": None}) + assert meta["user_api_key"] is None + + +# ── get_logging_payload key-hash invariant tests ─────────────────────────── + + +@patch("litellm.proxy.proxy_server.master_key", None) +@patch("litellm.proxy.proxy_server.general_settings", {}) +def test_get_logging_payload_non_sk_raw_key_both_fields_hashed(): + raw = "anthropic-raw-key-xyz" + kwargs = { + "model": "openai/gpt-4.1", + "messages": [{"role": "user", "content": "Hello"}], + "call_type": "acompletion", + "litellm_params": { + "metadata": { + "user_api_key": raw, + "user_api_key_user_id": "test_user", + "user_api_key_team_id": "test_team", + } + }, + } + payload = get_logging_payload( + kwargs=kwargs, + response_obj=Exception("error"), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert payload["api_key"] != raw + assert len(payload["api_key"]) == 64 + + parsed_meta = json.loads(payload["metadata"]) + assert parsed_meta["user_api_key"] != raw + assert parsed_meta["user_api_key"] is not None + assert len(parsed_meta["user_api_key"]) == 64 + + +@patch("litellm.proxy.proxy_server.master_key", None) +@patch("litellm.proxy.proxy_server.general_settings", {}) +def test_get_logging_payload_empty_key_slp_none_is_empty_string_not_none_literal(): + kwargs = { + "model": "openai/gpt-4.1", + "messages": [{"role": "user", "content": "Hello"}], + "call_type": "acompletion", + "litellm_params": { + "metadata": { + "user_api_key_user_id": "test_user", + } + }, + } + payload = get_logging_payload( + kwargs=kwargs, + response_obj=Exception("error"), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert payload["api_key"] == "", ( + f"Expected empty string but got {payload['api_key']!r}; " + "dropping _redact_logged_api_key's 'or \"\"' guard would yield 'None' here" + ) + + +def test_get_spend_logs_metadata_sibling_fields_preserved(): + raw = "anthropic-raw-key-xyz" + meta = _get_spend_logs_metadata( + { + "user_api_key": raw, + "user_api_key_alias": "my-alias", + "user_api_key_team_id": "team-123", + } + ) + assert meta["user_api_key"] == hash_token(raw) + assert meta["user_api_key_alias"] == "my-alias" + assert meta["user_api_key_team_id"] == "team-123" + + +def test_redact_logged_api_key_partial_sha256_is_hashed(): + partial_hex = "a" * 63 + result = _redact_logged_api_key(partial_hex) + assert result is not None + assert result != partial_hex + assert len(result) == 64 + assert result == hash_token(partial_hex) + + +def test_redact_logged_api_key_bearer_already_hashed_passes_through_with_flag(): + already_hashed = hash_token("sk-some-key") + assert len(already_hashed) == 64 + result = _redact_logged_api_key(f"Bearer {already_hashed}", already_hashed=True) + assert result == already_hashed + assert hash_token(already_hashed) != result + + +def test_redact_logged_api_key_bearer_sha256_without_flag_is_hashed(): + already_hashed = hash_token("sk-some-key") + assert len(already_hashed) == 64 + result = _redact_logged_api_key(f"Bearer {already_hashed}") + assert result is not None + assert result != already_hashed + assert result == hash_token(already_hashed) From 9c922f4aa4f6ab3a261326332d01b74e3717e63c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 15 Aug 2026 02:49:10 +0000 Subject: [PATCH 02/12] fix(bedrock): forward provider response headers on chat completions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/streaming_handler.py | 2 +- litellm/llms/bedrock/chat/converse_handler.py | 18 ++++-- litellm/llms/bedrock/chat/invoke_handler.py | 8 +-- .../base_invoke_transformation.py | 64 +++++++++---------- litellm/llms/custom_httpx/llm_http_handler.py | 2 + litellm/types/utils.py | 7 +- .../llm_translation/test_bedrock_moonshot.py | 19 ++---- .../llms/bedrock/chat/test_invoke_handler.py | 25 ++++++++ .../llms/chat/test_converse_handler.py | 55 ++++++++++++++++ 9 files changed, 143 insertions(+), 57 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 99b1c1a2ab7..fe107d349c6 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -164,7 +164,7 @@ class CustomStreamWrapper: custom_llm_provider: str | None = None, stream_options=None, make_call: Callable | None = None, - _response_headers: dict | None = None, + _response_headers: dict | httpx.Headers | None = None, ): self.model = model self.make_call = make_call diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 25e544f4521..c9ea06d37dc 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -33,7 +33,7 @@ def make_sync_call( json_mode: bool | None = False, fake_stream: bool = False, stream_chunk_size: int | None = None, -): +) -> tuple[Any, httpx.Headers]: if client is None: client = _get_httpx_client() # Create a new client if none provided @@ -74,7 +74,7 @@ def make_sync_call( additional_args={"complete_input_dict": data}, ) - return completion_stream + return completion_stream, response.headers class BedrockConverseLLM(BaseAWSLLM): @@ -132,7 +132,7 @@ class BedrockConverseLLM(BaseAWSLLM): }, ) - completion_stream: Final = await make_call( + completion_stream, response_headers = await make_call( client=client, api_base=api_base, headers=dict(prepped.headers), @@ -149,6 +149,7 @@ class BedrockConverseLLM(BaseAWSLLM): model=model, custom_llm_provider="bedrock", logging_obj=logging_obj, + _response_headers=response_headers, ) return streaming_response @@ -225,7 +226,7 @@ class BedrockConverseLLM(BaseAWSLLM): except httpx.TimeoutException: raise BedrockError(status_code=408, message="Timeout error occurred.") - return litellm.AmazonConverseConfig()._transform_response( + transformed_response: Final = litellm.AmazonConverseConfig()._transform_response( model=model, response=response, model_response=model_response, @@ -237,6 +238,8 @@ class BedrockConverseLLM(BaseAWSLLM): optional_params=optional_params, encoding=encoding, ) + transformed_response.set_provider_response_headers(response.headers) + return transformed_response def completion( self, @@ -440,7 +443,7 @@ class BedrockConverseLLM(BaseAWSLLM): client = client if stream is not None and stream is True: - completion_stream: Final = make_sync_call( + completion_stream, response_headers = make_sync_call( client=(client if client is not None and isinstance(client, HTTPHandler) else None), api_base=proxy_endpoint_url, headers=prepped.headers, @@ -457,6 +460,7 @@ class BedrockConverseLLM(BaseAWSLLM): model=model, custom_llm_provider="bedrock", logging_obj=logging_obj, + _response_headers=response_headers, ) return streaming_response @@ -477,7 +481,7 @@ class BedrockConverseLLM(BaseAWSLLM): except httpx.TimeoutException: raise BedrockError(status_code=408, message="Timeout error occurred.") - return litellm.AmazonConverseConfig()._transform_response( + sync_transformed_response: Final = litellm.AmazonConverseConfig()._transform_response( model=model, response=response, model_response=model_response, @@ -489,3 +493,5 @@ class BedrockConverseLLM(BaseAWSLLM): optional_params=optional_params, encoding=encoding, ) + sync_transformed_response.set_provider_response_headers(response.headers) + return sync_transformed_response diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 8d2b3dae71b..21892af3aae 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -163,7 +163,7 @@ async def make_call( json_mode: bool | None = False, bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None, stream_chunk_size: int | None = None, -): +) -> tuple[Any, httpx.Headers]: try: if client is None: client = get_async_httpx_client( @@ -225,7 +225,7 @@ async def make_call( additional_args={"complete_input_dict": data}, ) - return completion_stream + return completion_stream, response.headers except httpx.HTTPStatusError as err: error_code: Final = err.response.status_code raise BedrockError(status_code=error_code, message=err.response.text) @@ -248,7 +248,7 @@ def make_sync_call( json_mode: bool | None = False, bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None, stream_chunk_size: int | None = None, -): +) -> tuple[Any, httpx.Headers]: try: if client is None: client = _get_httpx_client( @@ -309,7 +309,7 @@ def make_sync_call( additional_args={"complete_input_dict": data}, ) - return completion_stream + return completion_stream, response.headers except httpx.HTTPStatusError as err: error_code: Final = err.response.status_code raise BedrockError(status_code=error_code, message=err.response.text) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 430d0a92b51..0b5855ccbd4 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -1,7 +1,6 @@ import copy import json import time -from functools import partial from typing import TYPE_CHECKING, Any, Final, cast, get_args import httpx @@ -444,24 +443,24 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): json_mode: bool | None = None, signed_json_body: bytes | None = None, ) -> CustomStreamWrapper: + completion_stream, response_headers = await make_call( + client=client, + api_base=api_base, + headers=headers, + data=json.dumps(data), + model=model, + messages=messages, + logging_obj=logging_obj, + fake_stream=True if "ai21" in api_base else False, + bedrock_invoke_provider=self.get_bedrock_invoke_provider(model), + json_mode=json_mode, + ) streaming_response: Final = CustomStreamWrapper( - completion_stream=None, - make_call=partial( - make_call, - client=client, - api_base=api_base, - headers=headers, - data=json.dumps(data), - model=model, - messages=messages, - logging_obj=logging_obj, - fake_stream=True if "ai21" in api_base else False, - bedrock_invoke_provider=self.get_bedrock_invoke_provider(model), - json_mode=json_mode, - ), + completion_stream=completion_stream, model=model, custom_llm_provider="bedrock", logging_obj=logging_obj, + _response_headers=response_headers, ) return streaming_response @@ -479,27 +478,28 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): json_mode: bool | None = None, signed_json_body: bytes | None = None, ) -> CustomStreamWrapper: - if client is None or isinstance(client, AsyncHTTPHandler): - client = _get_httpx_client(params={}) + sync_client: Final = ( + _get_httpx_client(params={}) if client is None or isinstance(client, AsyncHTTPHandler) else client + ) + completion_stream, response_headers = make_sync_call( + client=sync_client, + api_base=api_base, + headers=headers, + data=json.dumps(data), + signed_json_body=signed_json_body, + model=model, + messages=messages, + logging_obj=logging_obj, + fake_stream=True if "ai21" in api_base else False, + bedrock_invoke_provider=self.get_bedrock_invoke_provider(model), + json_mode=json_mode, + ) streaming_response: Final = CustomStreamWrapper( - completion_stream=None, - make_call=partial( - make_sync_call, - client=client, - api_base=api_base, - headers=headers, - data=json.dumps(data), - signed_json_body=signed_json_body, - model=model, - messages=messages, - logging_obj=logging_obj, - fake_stream=True if "ai21" in api_base else False, - bedrock_invoke_provider=self.get_bedrock_invoke_provider(model), - json_mode=json_mode, - ), + completion_stream=completion_stream, model=model, custom_llm_provider="bedrock", logging_obj=logging_obj, + _response_headers=response_headers, ) return streaming_response diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 721b9545ac1..36bcf58a243 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -608,6 +608,7 @@ class BaseLLMHTTPHandler: model=model, custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, + _response_headers=headers, ) if client is None or not isinstance(client, HTTPHandler): @@ -771,6 +772,7 @@ class BaseLLMHTTPHandler: model=model, custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, + _response_headers=_response_headers, ) return streamwrapper diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 220826ccbca..f49b5735f04 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -11,6 +11,7 @@ from typing import ( get_args, ) +import httpx from openai._models import BaseModel as OpenAIObject from openai.types.audio.transcription_create_params import ( FileTypes as FileTypes, @@ -49,7 +50,7 @@ from litellm.types.llms.base import ( ) from litellm.types.mcp import MCPServerCostInfo -from ..litellm_core_utils.core_helpers import map_finish_reason +from ..litellm_core_utils.core_helpers import map_finish_reason, process_response_headers from .agents import LiteLLMSendMessageResponse from .guardrails import GuardrailEventHooks from .llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse @@ -1896,6 +1897,10 @@ class ModelResponseBase(OpenAIObject): _response_headers: dict | None = None + def set_provider_response_headers(self, headers: httpx.Headers) -> None: + """Surface a provider's raw response headers to the caller as `llm_provider-*` headers.""" + self._hidden_params["additional_headers"] = process_response_headers(headers) + def model_dump(self, **kwargs): """Default to exclude_unset to avoid Pydantic serializer warnings for OpenAIObject-derived types.""" if "exclude_unset" not in kwargs and "exclude_none" not in kwargs: diff --git a/tests/llm_translation/test_bedrock_moonshot.py b/tests/llm_translation/test_bedrock_moonshot.py index a9f4a86b3b6..c777305d562 100644 --- a/tests/llm_translation/test_bedrock_moonshot.py +++ b/tests/llm_translation/test_bedrock_moonshot.py @@ -209,14 +209,12 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest): not exercised here — moonshot streaming delegates to the OpenAI parser and is covered by the OpenAI test suite. - Note: bedrock invoke streaming cannot be intercepted by patching - the caller-supplied client, because ``CustomStreamWrapper.fetch_sync_stream`` - at streaming_handler.py invokes the stored ``make_call`` partial with - ``client=litellm.module_level_client``, which overrides any client the - caller passed. Patch ``make_sync_call`` at its import site in - ``base_invoke_transformation`` so we observe the exact kwargs the - partial was built with at stream-wrapper construction time. + Patch ``make_sync_call`` at its import site in + ``base_invoke_transformation`` so we observe the exact kwargs it is + called with at stream-wrapper construction time. """ + import httpx + from litellm.utils import CustomStreamWrapper captured: dict = {} @@ -225,7 +223,7 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest): captured.update(kwargs) # Return an empty iterator so the stream wrapper's iteration # doesn't try to parse real bytes. - return iter([]) + return iter([]), httpx.Headers() with patch( "litellm.llms.bedrock.chat.invoke_transformations." @@ -246,11 +244,6 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest): aws_region_name="us-west-2", ) assert isinstance(response, CustomStreamWrapper) - # Trigger fetch_sync_stream → make_call(...) → fake_make_sync_call. - try: - next(iter(response)) - except StopIteration: - pass assert captured, "make_sync_call was never invoked" assert captured["api_base"].endswith("/invoke-with-response-stream") diff --git a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py index ee50b9db015..f211cdac475 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py @@ -2,17 +2,20 @@ import os import sys from unittest.mock import AsyncMock, MagicMock +import httpx import pytest sys.path.insert( 0, os.path.abspath("../../../../..") ) # Adds the parent directory to the system path +import litellm from litellm.llms.bedrock.chat.invoke_handler import ( AWSEventStreamDecoder, make_call, make_sync_call, ) +from litellm.llms.custom_httpx.http_handler import HTTPHandler def test_transform_thinking_blocks_with_redacted_content(): @@ -293,3 +296,25 @@ def test_make_sync_call_honors_explicit_stream_chunk_size(): response.iter_bytes.assert_called_once_with(chunk_size=2048) + +def test_invoke_streaming_forwards_bedrock_response_headers(): + """Streaming callers need `x-amzn-requestid` to correlate a LiteLLM request with AWS support.""" + response = MagicMock() + response.status_code = 200 + response.iter_bytes = MagicMock(return_value=iter([])) + response.headers = httpx.Headers({"x-amzn-requestid": "req-789"}) + client = HTTPHandler() + client.post = MagicMock(return_value=response) + + stream = litellm.completion( + model="bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "hi"}], + stream=True, + client=client, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + + assert stream._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-789" + diff --git a/tests/test_litellm/llms/chat/test_converse_handler.py b/tests/test_litellm/llms/chat/test_converse_handler.py index 2a3db5982ef..89ab29122d7 100644 --- a/tests/test_litellm/llms/chat/test_converse_handler.py +++ b/tests/test_litellm/llms/chat/test_converse_handler.py @@ -1,7 +1,9 @@ +import json import os import sys from unittest.mock import MagicMock +import httpx import pytest import litellm @@ -202,6 +204,59 @@ def test_make_sync_call_honors_explicit_stream_chunk_size(): response.iter_bytes.assert_called_once_with(chunk_size=2048) +def _converse_response_body() -> dict: + return { + "output": {"message": {"role": "assistant", "content": [{"text": "hi"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + } + + +def test_converse_completion_forwards_bedrock_response_headers(): + """Bedrock returns x-amzn-requestid on every converse call, which customers need to + correlate proxy requests with AWS support cases, so it must reach the caller as + llm_provider-x-amzn-requestid.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json = MagicMock(return_value=_converse_response_body()) + mock_response.text = json.dumps(_converse_response_body()) + mock_response.headers = httpx.Headers({"x-amzn-requestid": "req-123"}) + client = HTTPHandler() + client.post = MagicMock(return_value=mock_response) + + response = litellm.completion( + model="bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "hi"}], + client=client, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + + assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-123" + + +def test_converse_streaming_forwards_bedrock_response_headers(): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.iter_bytes = MagicMock(return_value=iter([])) + mock_response.headers = httpx.Headers({"x-amzn-requestid": "req-456"}) + client = HTTPHandler() + client.post = MagicMock(return_value=mock_response) + + response = litellm.completion( + model="bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "hi"}], + stream=True, + client=client, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + + assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-456" + + def test_completion_plumbs_stream_chunk_size_through_converse(): iter_bytes_spy = _stream_completion_with_spied_iter_bytes( model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" From 726db1a4c118a2ab92214ba741efacdce7fe25b5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 15 Aug 2026 03:07:14 +0000 Subject: [PATCH 03/12] test(bedrock): cover async header forwarding for converse and invoke Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_translation/test_bedrock_moonshot.py | 3 +- .../llms/bedrock/chat/test_invoke_handler.py | 29 +++++++++- .../llms/chat/test_converse_handler.py | 55 +++++++++++++++++-- 3 files changed, 78 insertions(+), 9 deletions(-) diff --git a/tests/llm_translation/test_bedrock_moonshot.py b/tests/llm_translation/test_bedrock_moonshot.py index c777305d562..61364cf2caa 100644 --- a/tests/llm_translation/test_bedrock_moonshot.py +++ b/tests/llm_translation/test_bedrock_moonshot.py @@ -12,6 +12,7 @@ This test suite verifies: """ from base_llm_unit_tests import BaseLLMChatTest +import httpx import pytest import sys import os @@ -213,8 +214,6 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest): ``base_invoke_transformation`` so we observe the exact kwargs it is called with at stream-wrapper construction time. """ - import httpx - from litellm.utils import CustomStreamWrapper captured: dict = {} diff --git a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py index f211cdac475..e8964910c69 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py @@ -15,7 +15,7 @@ from litellm.llms.bedrock.chat.invoke_handler import ( make_call, make_sync_call, ) -from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler def test_transform_thinking_blocks_with_redacted_content(): @@ -298,7 +298,6 @@ def test_make_sync_call_honors_explicit_stream_chunk_size(): def test_invoke_streaming_forwards_bedrock_response_headers(): - """Streaming callers need `x-amzn-requestid` to correlate a LiteLLM request with AWS support.""" response = MagicMock() response.status_code = 200 response.iter_bytes = MagicMock(return_value=iter([])) @@ -318,3 +317,29 @@ def test_invoke_streaming_forwards_bedrock_response_headers(): assert stream._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-789" + +@pytest.mark.asyncio +async def test_async_invoke_streaming_forwards_bedrock_response_headers(): + async def _no_bytes(chunk_size=None): + return + yield b"" + + response = MagicMock() + response.status_code = 200 + response.aiter_bytes = _no_bytes + response.headers = httpx.Headers({"x-amzn-requestid": "req-987"}) + client = AsyncHTTPHandler() + client.post = AsyncMock(return_value=response) + + stream = await litellm.acompletion( + model="bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "hi"}], + stream=True, + client=client, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + + assert stream._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-987" + diff --git a/tests/test_litellm/llms/chat/test_converse_handler.py b/tests/test_litellm/llms/chat/test_converse_handler.py index 89ab29122d7..6f8a2788c38 100644 --- a/tests/test_litellm/llms/chat/test_converse_handler.py +++ b/tests/test_litellm/llms/chat/test_converse_handler.py @@ -1,7 +1,7 @@ import json import os import sys -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock import httpx import pytest @@ -10,7 +10,7 @@ import litellm from litellm.llms.bedrock.chat import BedrockConverseLLM from litellm.llms.bedrock.chat.converse_handler import make_sync_call from litellm.llms.bedrock.common_utils import _get_all_bedrock_regions -from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler sys.path.insert( 0, os.path.abspath("../../../../..") @@ -213,9 +213,6 @@ def _converse_response_body() -> dict: def test_converse_completion_forwards_bedrock_response_headers(): - """Bedrock returns x-amzn-requestid on every converse call, which customers need to - correlate proxy requests with AWS support cases, so it must reach the caller as - llm_provider-x-amzn-requestid.""" mock_response = MagicMock() mock_response.status_code = 200 mock_response.json = MagicMock(return_value=_converse_response_body()) @@ -257,6 +254,54 @@ def test_converse_streaming_forwards_bedrock_response_headers(): assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-456" +@pytest.mark.asyncio +async def test_async_converse_completion_forwards_bedrock_response_headers(): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json = MagicMock(return_value=_converse_response_body()) + mock_response.text = json.dumps(_converse_response_body()) + mock_response.headers = httpx.Headers({"x-amzn-requestid": "req-abc"}) + client = AsyncHTTPHandler() + client.post = AsyncMock(return_value=mock_response) + + response = await litellm.acompletion( + model="bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "hi"}], + client=client, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + + assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-abc" + + +@pytest.mark.asyncio +async def test_async_converse_streaming_forwards_bedrock_response_headers(): + async def _no_bytes(chunk_size=None): + return + yield b"" + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.aiter_bytes = _no_bytes + mock_response.headers = httpx.Headers({"x-amzn-requestid": "req-def"}) + client = AsyncHTTPHandler() + client.post = AsyncMock(return_value=mock_response) + + response = await litellm.acompletion( + model="bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "hi"}], + stream=True, + client=client, + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + + assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-def" + + def test_completion_plumbs_stream_chunk_size_through_converse(): iter_bytes_spy = _stream_completion_with_spied_iter_bytes( model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" From 7d999a15864a2821dfd85f023c5064fe497eb897 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 19:41:12 -0700 Subject: [PATCH 04/12] fix(cognition): price swe-1.7 at the standard tier, add swe-1.7-lightning The cost map shipped cognition/swe-1.7 at $2.50 in / $12.50 out per million with $1.00 cache reads. Those are the Lightning numbers. Cognition's own model list at https://docs.devin.ai/desktop/models has uid swe-1-7 at $0.50 / $2.50 with $0.20 cache reads, and uid swe-1-7-lightning at $2.50 / $12.50 with $1.00 cache reads, so every swe-1.7 call has been costed at 5x since the entry landed. swe-1.7 now carries the standard rates and the Lightning tier gets its own entry, in both cost map copies. The source field on both moves to the desktop models page, which is the one that lists both tiers. --- ...odel_prices_and_context_window_backup.json | 12 +++- model_prices_and_context_window.json | 12 +++- .../openai_like/test_cognition_provider.py | 58 ++++++++++++++++--- 3 files changed, 72 insertions(+), 10 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index efbe0d2ebb7..c5579f03a3e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48949,6 +48949,16 @@ "source": "https://docs.devin.ai/windsurf/plugins/cascade/models" }, "cognition/swe-1.7": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + "litellm_provider": "cognition", + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "source": "https://docs.devin.ai/desktop/models" + }, + "cognition/swe-1.7-lightning": { "input_cost_per_token": 2.5e-06, "output_cost_per_token": 1.25e-05, "cache_read_input_token_cost": 1e-06, @@ -48956,7 +48966,7 @@ "mode": "chat", "supports_function_calling": true, "supports_prompt_caching": true, - "source": "https://docs.devin.ai/windsurf/plugins/cascade/models" + "source": "https://docs.devin.ai/desktop/models" }, "pinstripes/ps/glm-4.5-air": { "max_tokens": 128000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index efbe0d2ebb7..c5579f03a3e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48949,6 +48949,16 @@ "source": "https://docs.devin.ai/windsurf/plugins/cascade/models" }, "cognition/swe-1.7": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 2e-07, + "litellm_provider": "cognition", + "mode": "chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "source": "https://docs.devin.ai/desktop/models" + }, + "cognition/swe-1.7-lightning": { "input_cost_per_token": 2.5e-06, "output_cost_per_token": 1.25e-05, "cache_read_input_token_cost": 1e-06, @@ -48956,7 +48966,7 @@ "mode": "chat", "supports_function_calling": true, "supports_prompt_caching": true, - "source": "https://docs.devin.ai/windsurf/plugins/cascade/models" + "source": "https://docs.devin.ai/desktop/models" }, "pinstripes/ps/glm-4.5-air": { "max_tokens": 128000, diff --git a/tests/test_litellm/llms/openai_like/test_cognition_provider.py b/tests/test_litellm/llms/openai_like/test_cognition_provider.py index c358d178f60..5c71b60e08a 100644 --- a/tests/test_litellm/llms/openai_like/test_cognition_provider.py +++ b/tests/test_litellm/llms/openai_like/test_cognition_provider.py @@ -111,33 +111,51 @@ class TestCognitionProviderIdentity: class TestCognitionCostTracking: @pytest.mark.parametrize( - "model, input_cost, output_cost", + "model, input_cost, output_cost, cache_read_cost", [ - ("cognition/swe-1.6", 5e-07, 2.5e-06), - ("cognition/swe-1.7", 2.5e-06, 1.25e-05), + ("cognition/swe-1.6", 5e-07, 2.5e-06, 2e-07), + ("cognition/swe-1.7", 5e-07, 2.5e-06, 2e-07), + ("cognition/swe-1.7-lightning", 2.5e-06, 1.25e-05, 1e-06), ], ) - def test_cost_map_entries(self, model: str, input_cost: float, output_cost: float): + def test_cost_map_entries(self, model: str, input_cost: float, output_cost: float, cache_read_cost: float): info = litellm.get_model_info(model=model) assert info["litellm_provider"] == "cognition" assert info["mode"] == "chat" assert info["input_cost_per_token"] == input_cost assert info["output_cost_per_token"] == output_cost + assert info["cache_read_input_token_cost"] == cache_read_cost - def test_cost_differs_from_openai_pricing(self): + @pytest.mark.parametrize( + "model, expected_prompt_cost, expected_completion_cost", + [ + ("cognition/swe-1.7", 0.5, 2.5), + ("cognition/swe-1.7-lightning", 2.5, 12.5), + ], + ) + def test_cost_differs_from_openai_pricing( + self, model: str, expected_prompt_cost: float, expected_completion_cost: float + ): """A cognition-prefixed model must never be priced off an OpenAI cost entry.""" from litellm.cost_calculator import cost_per_token prompt_cost, completion_cost = cost_per_token( - model="cognition/swe-1.7", + model=model, prompt_tokens=1_000_000, completion_tokens=1_000_000, custom_llm_provider="cognition", ) - assert prompt_cost == pytest.approx(2.5) - assert completion_cost == pytest.approx(12.5) + assert prompt_cost == pytest.approx(expected_prompt_cost) + assert completion_cost == pytest.approx(expected_completion_cost) + + def test_lightning_is_five_times_the_standard_tier(self): + standard = litellm.get_model_info(model="cognition/swe-1.7") + lightning = litellm.get_model_info(model="cognition/swe-1.7-lightning") + + assert lightning["input_cost_per_token"] == pytest.approx(standard["input_cost_per_token"] * 5) + assert lightning["output_cost_per_token"] == pytest.approx(standard["output_cost_per_token"] * 5) def test_supported_endpoints_matrix(self): matrix = json.loads((Path(litellm.__file__).parent / "provider_endpoints_support_backup.json").read_text()) @@ -170,6 +188,30 @@ class TestCognitionRouting: mock_response="hello from swe", ) + usage = response.usage + expected = usage.prompt_tokens * 5e-07 + usage.completion_tokens * 2.5e-06 + assert response._hidden_params["response_cost"] == pytest.approx(expected) + + @pytest.mark.asyncio + async def test_router_spend_uses_the_lightning_entry_for_lightning(self): + """The Lightning tier is its own model, costed off its own entry.""" + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "swe-lightning", + "litellm_params": {"model": "cognition/swe-1.7-lightning", "api_key": "sk-test"}, + } + ] + ) + + response = await router.acompletion( + model="swe-lightning", + messages=[{"role": "user", "content": "hi"}], + mock_response="hello from swe lightning", + ) + usage = response.usage expected = usage.prompt_tokens * 2.5e-06 + usage.completion_tokens * 1.25e-05 assert response._hidden_params["response_cost"] == pytest.approx(expected) From 16cd08054fddd9effd266f07b251db82c909ad9f Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 20 Aug 2026 19:20:47 -0700 Subject: [PATCH 05/12] fix: populate team member emails missing from the roster snapshot `members_with_roles` is a denormalized JSON snapshot written at add-time. `_update_team_members_list` backfilled `user_id` from `user_email` but never the reverse, so a member added by `user_id` alone was stored with `user_email=None` permanently - and `/team/info` returns that blob verbatim with no join to `LiteLLM_UserTable`, so the Admin UI's member table renders "-" for a user that plainly has an email. Fix both ends: - write path: `_resolve_member_identity` resolves identity both ways off the user rows the add just touched, so new roster entries stop being born blank. - read path: `/team/info` fills blank emails from `LiteLLM_UserTable` in one indexed `user_id IN (...)` query, repairing rows already in the database. Members that already carry an email are passed through untouched and cost no query, so this only ever turns a null into the right value. --- .../management_endpoints/team_endpoints.py | 136 ++++++++---- .../test_team_endpoints.py | 194 ++++++++++++++++++ 2 files changed, 291 insertions(+), 39 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 82e22bb5bbf..a8e545a8551 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -2640,51 +2640,63 @@ async def _process_team_members( return updated_users, updated_team_memberships +def _resolve_member_identity(member: Member, updated_users: Sequence[LiteLLM_UserTable]) -> Member: + """Return ``member`` with whichever of ``user_id`` / ``user_email`` the caller left out filled in. + + The roster entry is a snapshot, so whatever is missing here is missing for good. + Resolution runs both ways off the user rows the add just touched: added by email + -> stamp the user_id, added by user_id -> stamp the email. A value the caller + supplied is never overwritten. + """ + resolved_user_id: Final = member.user_id or next( + ( + user.user_id + for user in updated_users + if member.user_email is not None and user.user_email == member.user_email + ), + None, + ) + resolved_user_email: Final = member.user_email or next( + ( + user.user_email + for user in updated_users + if resolved_user_id is not None and user.user_id == resolved_user_id and user.user_email is not None + ), + None, + ) + return member.model_copy( + update={ # mutable-ok: pydantic update payload + "user_id": resolved_user_id, + "user_email": resolved_user_email, + } + ) + + +def _member_already_in_team(member: Member, complete_team_data: LiteLLM_TeamTable) -> bool: + return any( + (member.user_id is not None and existing_member.user_id == member.user_id) + or (member.user_email is not None and existing_member.user_email == member.user_email) + for existing_member in complete_team_data.members_with_roles + ) + + async def _update_team_members_list( data: TeamMemberAddRequest, complete_team_data: LiteLLM_TeamTable, updated_users: list[LiteLLM_UserTable], ) -> None: """Update the team's members_with_roles list.""" - if isinstance(data.member, Member): - new_member: Final = data.member.model_copy() + requested_members: Final[Sequence[Member]] = ( + (data.member,) if isinstance(data.member, Member) else tuple(data.member) + ) + resolved_members: Final = tuple(_resolve_member_identity(m, updated_users) for m in requested_members) - # get user id - if new_member.user_id is None and new_member.user_email is not None: - for user in updated_users: - if user.user_email is not None and user.user_email == new_member.user_email: - new_member.user_id = user.user_id - - # Check if member already exists in team before adding - member_already_exists = False - for existing_member in complete_team_data.members_with_roles: - if (new_member.user_id is not None and existing_member.user_id == new_member.user_id) or ( - new_member.user_email is not None and existing_member.user_email == new_member.user_email - ): - member_already_exists = True - break - - if not member_already_exists: - complete_team_data.members_with_roles.append(new_member) - - elif isinstance(data.member, list): - for nm in data.member: - if nm.user_id is None and nm.user_email is not None: - for user in updated_users: - if user.user_email is not None and user.user_email == nm.user_email: - nm.user_id = user.user_id - - # Check if member already exists in team before adding - member_already_exists = False - for existing_member in complete_team_data.members_with_roles: - if (nm.user_id is not None and existing_member.user_id == nm.user_id) or ( - nm.user_email is not None and existing_member.user_email == nm.user_email - ): - member_already_exists = True - break - - if not member_already_exists: - complete_team_data.members_with_roles.append(nm) + # extend() consumes the generator as it appends, so a member already added by this + # same call is seen by the next _member_already_in_team check - the batch dedupes + # against itself exactly as the append-one-at-a-time loop this replaced did. + complete_team_data.members_with_roles.extend( # rebind-ok: this helper's contract is to grow the caller's roster in place + m for m in resolved_members if not _member_already_in_team(m, complete_team_data) + ) async def _add_team_members_to_team( @@ -4086,6 +4098,39 @@ async def _add_team_member_budget_table( return team_info_response_object +async def _hydrate_member_emails( + prisma_client: PrismaClient, + members: Sequence[Member], +) -> tuple[Member, ...]: + """Fill in ``user_email`` for roster entries that were stored without one. + + ``members_with_roles`` is a denormalized snapshot written at add-time, so an entry + stored with ``user_email=None`` keeps that null even once the user row has an email. + Look the missing ones up in ``LiteLLM_UserTable`` (one indexed query) and fill them + in. A stored email is never overwritten - the snapshot stays the source of truth + wherever it has a value. + """ + missing_user_ids: Final = frozenset(m.user_id for m in members if not m.user_email and m.user_id is not None) + if not missing_user_ids: + return tuple(members) + + user_rows: Final[Sequence[LiteLLM_UserTable]] = await _user_db(prisma_client).find_many( + where={ # mutable-ok: Prisma query filters are dict-shaped + "user_id": { # mutable-ok: Prisma query filters are dict-shaped + "in": sorted(missing_user_ids) + } + } + ) + email_by_user_id: Final = MappingProxyType({u.user_id: u.user_email for u in user_rows if u.user_email}) + + return tuple( + m.model_copy(update={"user_email": email_by_user_id[m.user_id]}) # mutable-ok: pydantic update payload + if not m.user_email and m.user_id in email_by_user_id + else m + for m in members + ) + + async def _resolve_team_access_group_resources( _team_info: TeamInfoResponseObjectTeamTable, ) -> TeamInfoResponseObjectTeamTable: @@ -4221,9 +4266,22 @@ async def team_info( # Resolve resources inherited from access groups resolved_team_info: Final = await _resolve_team_access_group_resources(_team_info) + # Fill in emails the add-time roster snapshot never captured + hydrated_members: Final = await _hydrate_member_emails( + prisma_client=prisma_client, + members=resolved_team_info.members_with_roles, + ) + hydrated_team_info: Final = resolved_team_info.model_copy( + update={ # mutable-ok: pydantic update payload + # list(), not the tuple: model_copy skips validation, so the field has + # to be handed the list[Member] the response model declares. + "members_with_roles": list(hydrated_members) # mutable-ok: declared list[Member] + } + ) + response_object: Final = TeamInfoResponseObject( team_id=team_id, - team_info=resolved_team_info, + team_info=hydrated_team_info, keys=keys, team_memberships=returned_tm, ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 0767288d0bc..e39b09ae073 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -10253,6 +10253,65 @@ async def test_team_info_returns_model_aliases(): assert litellm_model_table.model_aliases == {"gpt-4o": "gpt-4o-team-1"} +@pytest.mark.asyncio +async def test_team_info_hydrates_member_emails_from_the_user_table(): + """/team/info must fill in emails missing from the members_with_roles snapshot. + + members_with_roles is written at add-time, so a member added by user_id alone + carries user_email=None forever. Without this join the Admin UI's member table + shows "-" for a user that has an email on their user row. A stored email is left + exactly as-is. + """ + from fastapi import Request + + from litellm.proxy.management_endpoints import team_endpoints + + team_row = LiteLLM_TeamTable( + team_id="team-1", + members_with_roles=[ + Member(user_id="no-email-on-roster", role="admin"), + Member(user_id="already-stored", user_email="stored@example.com", role="user"), + ], + ) + + mock_prisma = MagicMock() + mock_prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=team_row) + mock_prisma.get_data = AsyncMock(return_value=[]) + + find_many = AsyncMock( + return_value=[ + LiteLLM_UserTable( + user_id="no-email-on-roster", + user_email="real@example.com", + max_budget=None, + spend=0.0, + models=[], + ) + ] + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch.object(team_endpoints, "get_all_team_memberships", AsyncMock(return_value=[])), + patch.object(team_endpoints, "UserRepository") as repo, + ): + repo.return_value.table.find_many = find_many + + response = await team_endpoints.team_info( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + members = response["team_info"].members_with_roles + assert [(m.user_id, m.user_email) for m in members] == [ + ("no-email-on-roster", "real@example.com"), + ("already-stored", "stored@example.com"), + ] + # only the member actually missing an email is looked up + assert find_many.await_args.kwargs["where"] == {"user_id": {"in": ["no-email-on-roster"]}} + + @pytest.mark.asyncio async def test_update_model_table_clears_aliases_with_empty_map(): """``model_aliases={}`` on /team/update must persist an empty map (json.dumps({})) @@ -11369,6 +11428,141 @@ async def test_resolve_existing_member_user_ids_skips_the_query_when_no_user_ids repo.return_value.table.find_many.assert_not_awaited() +def _user_row(user_id: str, user_email: str | None) -> LiteLLM_UserTable: + return LiteLLM_UserTable( + user_id=user_id, user_email=user_email, max_budget=None, spend=0.0, models=[] + ) + + +@pytest.mark.asyncio +async def test_hydrate_member_emails_fills_in_emails_the_roster_snapshot_never_captured(): + """A member added by user_id alone has user_email=None on the stored roster entry. + + /team/info has to fill it in from the user row, or the UI renders "-" for a user + that plainly has an email. + """ + from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_emails + + find_many = AsyncMock(return_value=[_user_row("by-id", "found@example.com")]) + + with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: + repo.return_value.table.find_many = find_many + + hydrated = await _hydrate_member_emails( + prisma_client=MagicMock(), + members=[Member(user_id="by-id", role="admin")], + ) + + assert [(m.user_id, m.user_email, m.role) for m in hydrated] == [("by-id", "found@example.com", "admin")] + find_many.assert_awaited_once() + assert find_many.await_args.kwargs["where"] == {"user_id": {"in": ["by-id"]}} + + +@pytest.mark.asyncio +async def test_hydrate_member_emails_never_overwrites_a_stored_email(): + """The snapshot wins wherever it has a value - hydration only fills blanks. + + Overwriting would be a real behavior change to /team/info; filling a null is not. + """ + from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_emails + + find_many = AsyncMock(return_value=[_user_row("has-email", "current@example.com")]) + + with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: + repo.return_value.table.find_many = find_many + + hydrated = await _hydrate_member_emails( + prisma_client=MagicMock(), + members=[Member(user_id="has-email", user_email="stored@example.com", role="user")], + ) + + assert hydrated[0].user_email == "stored@example.com" + # nothing was missing, so no round-trip either + find_many.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_hydrate_member_emails_leaves_members_alone_when_the_user_row_has_no_email(): + """A user row with no email leaves the member as-is rather than inventing one.""" + from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_emails + + with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: + repo.return_value.table.find_many = AsyncMock(return_value=[_user_row("no-email", None)]) + + hydrated = await _hydrate_member_emails( + prisma_client=MagicMock(), + members=[Member(user_id="no-email", role="user"), Member(user_email="e@example.com", role="user")], + ) + + assert [m.user_email for m in hydrated] == [None, "e@example.com"] + + +@pytest.mark.asyncio +async def test_hydrate_member_emails_skips_the_query_when_every_member_has_one(): + """No blanks means /team/info pays for no extra query.""" + from litellm.proxy.management_endpoints.team_endpoints import _hydrate_member_emails + + with patch("litellm.proxy.management_endpoints.team_endpoints.UserRepository") as repo: + repo.return_value.table.find_many = AsyncMock() + + hydrated = await _hydrate_member_emails( + prisma_client=MagicMock(), + members=[Member(user_id="a", user_email="a@example.com", role="user")], + ) + + assert hydrated[0].user_email == "a@example.com" + repo.return_value.table.find_many.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_update_team_members_list_stamps_email_for_a_member_added_by_user_id(): + """Identity resolution runs both ways, so new roster entries stop being born blank. + + Previously only user_id was backfilled (from email); a member added by user_id + was written with user_email=None forever. + """ + from litellm.proxy.management_endpoints.team_endpoints import ( + _update_team_members_list, + ) + + mock_team = MagicMock(spec=LiteLLM_TeamTable) + mock_team.members_with_roles = [] + + await _update_team_members_list( + data=TeamMemberAddRequest(team_id="test-team-123", member=Member(user_id="new-user-123", role="user")), + complete_team_data=mock_team, + updated_users=[_user_row("new-user-123", "new@example.com")], + ) + + assert len(mock_team.members_with_roles) == 1 + assert mock_team.members_with_roles[0].user_email == "new@example.com" + + +@pytest.mark.asyncio +async def test_update_team_members_list_stamps_email_for_each_member_in_a_bulk_add(): + """Same both-ways resolution for the list branch.""" + from litellm.proxy.management_endpoints.team_endpoints import ( + _update_team_members_list, + ) + + mock_team = MagicMock(spec=LiteLLM_TeamTable) + mock_team.members_with_roles = [] + + await _update_team_members_list( + data=TeamMemberAddRequest( + team_id="test-team-123", + member=[Member(user_id="u1", role="user"), Member(user_email="u2@example.com", role="admin")], + ), + complete_team_data=mock_team, + updated_users=[_user_row("u1", "u1@example.com"), _user_row("u2", "u2@example.com")], + ) + + assert [(m.user_id, m.user_email) for m in mock_team.members_with_roles] == [ + ("u1", "u1@example.com"), + ("u2", "u2@example.com"), + ] + + def test_pre_existing_user_ids_counts_ids_filled_in_by_member_resolution(): """An id the member-resolution step filled in came from a matched row, so it pre-existed. From 722c650bfdf5375fc9e2d444485156b97c085762 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 19:47:05 -0700 Subject: [PATCH 06/12] test: cover generic HTTP streaming provider header forwarding Add sync and async regression tests for the BaseLLMHTTPHandler streaming path, which forwards provider response headers for the ~30 providers that ride the generic handler and had no coverage. Also drop redundant setup prose from the moonshot invoke test docstring. --- .../llm_translation/test_bedrock_moonshot.py | 4 - .../custom_httpx/test_llm_http_handler.py | 76 +++++++++++++++++++ 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/tests/llm_translation/test_bedrock_moonshot.py b/tests/llm_translation/test_bedrock_moonshot.py index 61364cf2caa..a82d1c6f029 100644 --- a/tests/llm_translation/test_bedrock_moonshot.py +++ b/tests/llm_translation/test_bedrock_moonshot.py @@ -209,10 +209,6 @@ class TestBedrockMoonshotInvoke(BaseLLMChatTest): endpoint with the messages body. Iteration of the stream itself is not exercised here — moonshot streaming delegates to the OpenAI parser and is covered by the OpenAI test suite. - - Patch ``make_sync_call`` at its import site in - ``base_invoke_transformation`` so we observe the exact kwargs it is - called with at stream-wrapper construction time. """ from litellm.utils import CustomStreamWrapper diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index fddd8d09dfc..b78b313e05c 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -2071,3 +2071,79 @@ async def test_anthropic_invalid_thinking_signature_retry_resigns_bedrock_reques retry_authorization = posts[1]["headers"]["Authorization"] assert retry_authorization.startswith("AWS4-HMAC-SHA256") assert retry_authorization != first_attempt_headers["Authorization"] + + +_GENERIC_STREAM_SSE = ( + b'data: {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,' + b'"model":"test-model","choices":[{"index":0,"delta":{"content":"hi"},' + b'"finish_reason":null}]}\n\n' + b"data: [DONE]\n\n" +) + + +def _generic_stream_upstream_response() -> httpx.Response: + return httpx.Response( + 200, + headers={ + "x-request-id": "generic-req-123", + "x-ratelimit-remaining-requests": "42", + }, + content=_GENERIC_STREAM_SSE, + request=httpx.Request("POST", "https://fake-vllm.test/v1/chat/completions"), + ) + + +def test_generic_http_handler_sync_streaming_forwards_provider_response_headers(): + """ + Regression test for the generic BaseLLMHTTPHandler streaming path used by + ~30 providers (deepseek, groq, hosted_vllm, databricks, openrouter, ...). + + The sync `completion()` streaming branch builds the CustomStreamWrapper from + `make_sync_call`, which returns the upstream response headers alongside the + stream. Those headers must reach the caller as `llm_provider-*` entries in + `_hidden_params["additional_headers"]`, which is what the proxy merges into + the client-facing response headers. + """ + mock_client = Mock(spec=HTTPHandler) + mock_client.post = Mock(return_value=_generic_stream_upstream_response()) + + response = litellm.completion( + model="hosted_vllm/test-model", + messages=[{"role": "user", "content": "Hello"}], + api_base="https://fake-vllm.test/v1", + api_key="sk-test", + stream=True, + client=mock_client, + ) + + additional_headers = response._hidden_params["additional_headers"] + assert additional_headers["llm_provider-x-request-id"] == "generic-req-123" + assert additional_headers["llm_provider-x-ratelimit-remaining-requests"] == "42" + + assert "".join([chunk.choices[0].delta.content or "" for chunk in response]) == "hi" + + +@pytest.mark.asyncio +async def test_generic_http_handler_async_streaming_forwards_provider_response_headers(): + """ + Companion to the sync test above for `acompletion_stream_function`, which + builds its CustomStreamWrapper from `make_async_call_stream_helper`. + """ + mock_client = AsyncMock(spec=AsyncHTTPHandler) + mock_client.post = AsyncMock(return_value=_generic_stream_upstream_response()) + + response = await litellm.acompletion( + model="hosted_vllm/test-model", + messages=[{"role": "user", "content": "Hello"}], + api_base="https://fake-vllm.test/v1", + api_key="sk-test", + stream=True, + client=mock_client, + ) + + additional_headers = response._hidden_params["additional_headers"] + assert additional_headers["llm_provider-x-request-id"] == "generic-req-123" + assert additional_headers["llm_provider-x-ratelimit-remaining-requests"] == "42" + + collected = [chunk async for chunk in response] + assert "".join([chunk.choices[0].delta.content or "" for chunk in collected]) == "hi" From 2ea633d223e57777edf39edb2c3ca6b18c0266d2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 19:57:46 -0700 Subject: [PATCH 07/12] fix(sagemaker_chat): send the inference component header and honor hf_model_name sagemaker_chat never put X-Amzn-SageMaker-Inference-Component on the request, so any endpoint backed by inference components answered 400 INFERENCE_COMPONENT_NAME_MISSING and the call never reached the container. The legacy sagemaker provider has built that header from model_id since #8889, and this brings the chat provider in line. It goes on in validate_environment, which runs before the request is SigV4-signed, so the signature covers it The request body also always named the endpoint rather than the served model, which containers that validate the body's model answer with a 404. hf_model_name now becomes the body's model when it is set, and endpoints that do not set it keep sending exactly what they send today --- litellm/llms/sagemaker/chat/transformation.py | 25 +++++- .../test_sagemaker_chat_transformation.py | 84 +++++++++++++++++++ 2 files changed, 108 insertions(+), 1 deletion(-) diff --git a/litellm/llms/sagemaker/chat/transformation.py b/litellm/llms/sagemaker/chat/transformation.py index 99543e7add1..37ddd813d6f 100644 --- a/litellm/llms/sagemaker/chat/transformation.py +++ b/litellm/llms/sagemaker/chat/transformation.py @@ -54,7 +54,30 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM): api_key: str | None = None, api_base: str | None = None, ) -> dict: - return headers + inference_component_name: Final = optional_params.get("model_id") + if not isinstance(inference_component_name, str): + return headers + return {**headers, "X-Amzn-SageMaker-Inference-Component": inference_component_name} + + def transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: matches the base chat transform signature + optional_params: dict, # mutable-ok: matches the base chat transform signature + litellm_params: dict, # mutable-ok: matches the base chat transform signature + headers: dict, # mutable-ok: matches the base chat transform signature + ) -> dict: # mutable-ok: the handler sends this body straight to httpx + request: Final = super().transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + served_model_name: Final = litellm_params.get("hf_model_name") + if not isinstance(served_model_name, str): + return request + return {**request, "model": served_model_name} def get_complete_url( self, diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_chat_transformation.py b/tests/test_litellm/llms/sagemaker/test_sagemaker_chat_transformation.py index 54e6f95c795..da6caca4f05 100644 --- a/tests/test_litellm/llms/sagemaker/test_sagemaker_chat_transformation.py +++ b/tests/test_litellm/llms/sagemaker/test_sagemaker_chat_transformation.py @@ -19,6 +19,8 @@ from unittest.mock import MagicMock import httpx import pytest +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.sagemaker.chat.transformation import SagemakerChatConfig @@ -233,3 +235,85 @@ def test_decoder_reassembles_frames_across_arbitrary_byte_boundaries(split_size) ] assert texts == [f"token{i} " for i in range(len(frames))] + + +_INFERENCE_COMPONENT_HEADER = "X-Amzn-SageMaker-Inference-Component" + +_STUB_COMPLETION_RESPONSE = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1700000000, + "model": "served-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +class _RequestCapturingHTTPHandler(HTTPHandler): + """Injected transport that records exactly what sagemaker_chat put on the wire.""" + + def __init__(self) -> None: + super().__init__() + self.request_headers: dict[str, str] = {} + self.request_body: dict = {} + + def post(self, url: str, headers=None, data=None, **kwargs) -> httpx.Response: + self.request_headers = dict(headers or {}) + self.request_body = json.loads(data) + return httpx.Response(200, json=_STUB_COMPLETION_RESPONSE, request=httpx.Request("POST", url)) + + +def _invoke_sagemaker_chat(monkeypatch, **extra_params) -> _RequestCapturingHTTPHandler: + """Drive one sagemaker_chat completion against an injected transport. + + A Bedrock API key short-circuits SigV4 inside `BaseAWSLLM._sign_request`, which would hide + whether the inference-component header is really covered by the signature, so it is cleared. + """ + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + client = _RequestCapturingHTTPHandler() + litellm.completion( + model="sagemaker_chat/my-endpoint", + messages=[{"role": "user", "content": "hi"}], + aws_access_key_id="AKIATESTTESTTESTTEST", + aws_secret_access_key="test-secret-key", + aws_region_name="us-east-1", + client=client, + **extra_params, + ) + return client + + +def test_model_id_is_sent_as_a_signed_inference_component_header(monkeypatch): + """`model_id` names an inference component and must reach SageMaker as a signed header. + + Endpoints backed by inference components reject any request without + `X-Amzn-SageMaker-Inference-Component` with HTTP 400 INFERENCE_COMPONENT_NAME_MISSING, so the + header has to be built before `sign_request` runs and end up inside SignedHeaders. + """ + client = _invoke_sagemaker_chat(monkeypatch, model_id="my-inference-component") + + assert client.request_headers[_INFERENCE_COMPONENT_HEADER] == "my-inference-component" + assert "x-amzn-sagemaker-inference-component" in client.request_headers["Authorization"] + + +def test_no_inference_component_header_when_model_id_is_unset(monkeypatch): + """Plain endpoints must not receive the header at all, not even an empty one.""" + client = _invoke_sagemaker_chat(monkeypatch) + + assert not any(name.lower() == _INFERENCE_COMPONENT_HEADER.lower() for name in client.request_headers) + + +def test_hf_model_name_becomes_the_body_model(monkeypatch): + """`hf_model_name` names the served model, and containers that validate the body's `model` + 404 on the endpoint name, so it has to replace it rather than ride along as an extra field.""" + client = _invoke_sagemaker_chat(monkeypatch, hf_model_name="org/served-model") + + assert client.request_body["model"] == "org/served-model" + assert "hf_model_name" not in client.request_body + + +def test_body_model_stays_the_endpoint_name_when_hf_model_name_is_unset(monkeypatch): + """Without `hf_model_name` the body must keep the model it has today.""" + client = _invoke_sagemaker_chat(monkeypatch) + + assert client.request_body["model"] == "my-endpoint" From b76def0e5df1ce874c131fdbd4b4e10def90f3df Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 20 Aug 2026 20:24:49 -0700 Subject: [PATCH 08/12] test: require a `match=` on broad pytest.raises, and drop duplicate parametrize cases (#37769) `pytest.raises(Exception)` with no `match=` passes on any error that broad. A TypeError from a refactor, a botched fixture, an import that moved: all of them read as the rejection the test claims to police, so the test goes green for the wrong reason and stays green after the behaviour it guards is gone. PT011 closes that gap for the 317 sites B017 could not reach, because B017 only fires on a single-statement body with no `as e` binding. Each pattern here is the message the code actually raised, recorded by running the sites under a plugin that logged the concrete type and text per call site, so the assertions describe observed behaviour rather than a guess. Where a site raises more than one message across its parametrize cases, the pattern is an alternation of what was seen; where the exception carries an empty `str()` and puts the text on `.message`, the site keeps a narrow `noqa` with the reason. PT014 removes four parametrize cases that were listed twice. The duplicate re-runs an assertion that already passed, and it usually marks a case someone meant to vary and forgot to edit. --- ruff-tests.toml | 8 +- .../integrations/test_prometheus.py | 8 +- .../test_bedrock_apply_guardrail.py | 2 +- .../proxy/hooks/test_managed_files.py | 8 +- .../test_project_endpoints_prisma.py | 10 +- .../test_dynamoai_guardrails.py | 2 +- .../test_eu_ai_act_article5.py | 2 +- .../test_eu_ai_act_french_3_scenarios.py | 8 +- .../test_sg_mas_ai_guardrails.py | 2 +- .../test_sg_pdpa_guardrails.py | 2 +- tests/litellm_utils_tests/test_hashicorp.py | 2 +- tests/litellm_utils_tests/test_utils.py | 2 +- .../test_validate_tool_choice.py | 10 +- .../test_responses_hooks.py | 2 +- .../test_bedrock_completion.py | 2 +- .../test_convert_dict_to_chat_completion.py | 6 +- tests/llm_translation/test_prompt_factory.py | 6 +- tests/llm_translation/test_triton.py | 2 +- .../test_unit_test_bedrock_invoke.py | 2 +- tests/local_testing/test_auth_utils.py | 4 - tests/local_testing/test_exceptions.py | 2 +- tests/local_testing/test_file_types.py | 4 +- tests/local_testing/test_get_model_info.py | 1 - .../test_router_budget_limiter.py | 6 +- tests/local_testing/test_router_fallbacks.py | 2 +- .../test_standard_logging_payload.py | 2 +- .../test_update_team_e2e.py | 6 +- .../test_ocr_azure_document_intelligence.py | 2 +- tests/otel_tests/test_e2e_model_access.py | 9 +- .../test_key_management.py | 2 +- .../test_role_based_access.py | 4 +- tests/proxy_unit_tests/test_auth_checks.py | 4 +- tests/proxy_unit_tests/test_jwt.py | 10 +- tests/proxy_unit_tests/test_proxy_utils.py | 4 +- tests/proxy_unit_tests/test_update_spend.py | 2 +- .../test_router_helper_utils.py | 4 +- .../test_mcp_servers.py | 2 +- .../send_emails/test_sendgrid_email.py | 2 +- .../proxy/test_managed_files_hook.py | 2 +- .../cloudzero/test_cloudzero_database.py | 2 +- .../cloudzero/test_cz_stream_api.py | 2 +- .../integrations/focus/test_focus_database.py | 2 +- .../integrations/focus/test_s3_destination.py | 2 +- .../integrations/gitlab/test_gitlab_client.py | 10 +- .../integrations/levo/test_levo.py | 2 +- .../integrations/otel/test_otel_v2_metrics.py | 2 +- .../integrations/test_langfuse.py | 2 +- ...llm_core_utils_prompt_templates_factory.py | 2 +- ...test_initialize_dynamic_callback_params.py | 4 +- .../litellm_core_utils/test_llm_judge.py | 2 +- .../test_streaming_handler.py | 4 +- .../litellm_core_utils/test_token_counter.py | 20 +- .../litellm_core_utils/test_url_utils.py | 4 +- ...st_aiml_image_generation_transformation.py | 2 +- .../messages/test_mcp_handler.py | 4 +- .../test_azure_ai_rerank_transformation.py | 4 +- .../llms/bedrock/test_base_aws_llm.py | 4 +- ...bedrock_mantle_responses_transformation.py | 6 +- .../test_bedrock_mantle_transformation.py | 4 +- .../chat/test_bytez_chat_transformation.py | 2 +- .../test_deepinfra_rerank_transformation.py | 8 +- .../test_fal_ai_nano_banana_transformation.py | 2 +- .../test_featherless_chat_transformation.py | 6 +- ...test_fireworks_ai_rerank_transformation.py | 2 +- .../test_gemini_image_edit_transformation.py | 2 +- .../llms/gemini/test_gemini_client_setup.py | 4 +- .../test_hosted_vllm_rerank_transformation.py | 2 +- .../chat/test_langflow_chat_transformation.py | 2 +- ...est_modelscope_image_gen_transformation.py | 6 +- .../chat/test_novita_chat_transformation.py | 2 +- .../oci/chat/test_oci_chat_transformation.py | 16 +- .../test_pg_vector_transformation.py | 4 +- .../test_recraft_image_edit_transformation.py | 2 +- .../test_recraft_image_gen_transformation.py | 6 +- .../test_stability_image_generation.py | 6 +- .../llms/tinyfish/test_tinyfish_search.py | 12 +- .../files/test_vertex_ai_files_integration.py | 2 +- .../vertex_ai/test_vertex_ai_common_utils.py | 2 +- ...est_volcengine_responses_transformation.py | 2 +- .../volcengine/test_volcengine_embedding.py | 2 +- .../test_voyage_rerank_transformation.py | 4 +- .../test_voyage_multimodal_embedding.py | 4 +- .../llms/xai/test_xai_key_fallback.py | 2 +- .../auth/test_user_api_key_auth_mcp.py | 2 +- .../mcp_server/test_discoverable_endpoints.py | 7 - .../mcp_server/test_short_mcp_tool_prefix.py | 2 +- .../proxy/auth/test_auth_checks.py | 4 +- .../test_auth_hot_path_network_requests.py | 1042 ++++++++--------- .../proxy/auth/test_auth_utils.py | 18 +- .../proxy/auth/test_handle_jwt.py | 18 +- .../proxy/auth/test_oauth2_proxy_hook.py | 2 +- .../proxy/auth/test_route_checks.py | 14 +- .../proxy/auth/test_user_api_key_auth.py | 4 +- .../proxy/client/cli/test_auth_commands.py | 10 +- .../proxy/client/cli/test_pkce_login.py | 2 +- .../test_litellm/proxy/client/test_models.py | 4 +- .../proxy/common_utils/test_callback_utils.py | 6 +- .../proxy/common_utils/test_path_utils.py | 2 +- .../proxy/common_utils/test_timezone_utils.py | 6 +- .../test_spend_logs_partition_manager.py | 4 +- .../db/test_prisma_planned_engine_restart.py | 2 +- .../proxy/db/test_spend_log_tool_index.py | 2 +- .../test_bedrock_invoke_guardrail_checks.py | 4 +- .../guardrail_hooks/test_enkryptai.py | 2 +- .../test_generic_guardrail_api.py | 6 +- .../guardrail_hooks/test_model_armor.py | 6 +- .../guardrail_hooks/test_panw_prisma_airs.py | 2 +- .../guardrail_hooks/test_straiker.py | 6 +- .../guardrail_hooks/test_tool_permission.py | 4 +- .../proxy/guardrails/test_llm_as_a_judge.py | 2 +- .../hooks/test_sensitive_data_routing.py | 2 +- .../proxy/hooks/test_tpm_concurrent.py | 28 +- .../test_key_management_endpoints.py | 2 +- .../test_mcp_management_endpoints.py | 2 +- .../test_model_management_endpoints.py | 12 +- .../test_ptu_model_settings.py | 18 +- .../usage_endpoints/test_ai_usage_chat.py | 2 +- .../test_team_metadata_validation.py | 2 +- .../test_batch_guardrails.py | 2 +- .../test_llm_pass_through_endpoints.py | 8 +- .../policy_engine/test_policy_versioning.py | 4 +- .../proxy/proxy_server/test_proxy_config.py | 4 +- .../test_spend_management_endpoints.py | 4 +- .../proxy/test_common_request_processing.py | 2 +- .../proxy/test_enforce_user_param.py | 6 +- .../proxy/test_litellm_pre_call_utils.py | 6 +- tests/test_litellm/proxy/test_proxy_server.py | 2 +- .../proxy/test_spend_log_cleanup.py | 4 +- .../test_litellm/proxy/test_team_org_move.py | 2 +- .../proxy/utils/helpers/test_team_configs.py | 2 +- .../test_proxy_update_spend.py | 2 +- .../test_callback_capabilities_class.py | 2 +- .../test_responses_websocket_all_providers.py | 2 +- .../adaptive_router/test_bandit.py | 2 +- .../test_router_tag_routing.py | 26 +- .../test_litellm/sandbox/test_e2b_sandbox.py | 2 +- .../test_base_secret_manager.py | 2 +- .../test_github_close_low_quality_prs.py | 2 +- .../test_github_triage_with_llm.py | 4 +- .../test_redact_string_in_error_paths.py | 2 +- tests/test_litellm/test_redis.py | 4 +- tests/test_litellm/test_router.py | 10 +- .../test_router_model_cost_isolation.py | 4 +- tests/test_litellm/test_utils.py | 2 +- tests/test_litellm/types/test_router.py | 2 +- 145 files changed, 844 insertions(+), 867 deletions(-) diff --git a/ruff-tests.toml b/ruff-tests.toml index 6e77f4792a7..60438d355f0 100644 --- a/ruff-tests.toml +++ b/ruff-tests.toml @@ -20,6 +20,12 @@ # PT012 a `pytest.raises` block that runs on past the raising call. Everything after # that call is dead, so an `assert` sitting there is never checked. Keep the # block to the call itself and put the assertions below it +# PT011 `pytest.raises(Exception)` / `(ValueError)` / `(OSError)` with no `match=`. The +# block passes on any error that broad, so the TypeError a refactor introduced +# reads as the rejection under test. Pin the message the code actually raises +# PT014 the same `parametrize` case listed twice. The copy re-runs an assertion that +# already passed and adds no coverage, and it usually marks a case someone meant +# to vary and forgot to edit # # No target-version here on purpose: it resolves from requires-python (>=3.10), so # 3.11-only builtins like BaseExceptionGroup are correctly flagged in a tree that @@ -27,4 +33,4 @@ line-length = 120 -lint.select = ["F821", "B011", "B015", "B017", "B018", "PT012", "PT015", "PLR0133", "PLW0127"] +lint.select = ["F821", "B011", "B015", "B017", "B018", "PT011", "PT012", "PT014", "PT015", "PLR0133", "PLW0127"] diff --git a/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py b/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py index 0cd6055e09d..bdf73b6ab03 100644 --- a/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py +++ b/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py @@ -400,7 +400,7 @@ def test_invalid_metric_name_validation(): litellm.prometheus_metrics_config = test_config # Creating PrometheusLogger should raise ValueError due to invalid metric - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Configuration validation failed') as exc_info: PrometheusLogger() # Verify error message contains information about invalid metric @@ -429,7 +429,7 @@ def test_invalid_labels_validation(): litellm.prometheus_metrics_config = test_config # Creating PrometheusLogger should raise ValueError due to invalid labels - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Configuration validation failed') as exc_info: PrometheusLogger() # Verify error message contains information about invalid labels @@ -598,7 +598,7 @@ def test_invalid_exclude_metric_name_raises(reset_prometheus_exclude_settings): litellm.prometheus_exclude_labels = None litellm.prometheus_exclude_metrics = ["not_a_real_metric"] - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Prometheus exclude configuration validation failed') as exc_info: PrometheusLogger() assert "not_a_real_metric" in str(exc_info.value) @@ -612,7 +612,7 @@ def test_invalid_exclude_label_name_raises(reset_prometheus_exclude_settings): litellm.prometheus_exclude_metrics = None litellm.prometheus_exclude_labels = ["not_a_real_label"] - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Prometheus exclude configuration validation failed') as exc_info: PrometheusLogger() assert "not_a_real_label" in str(exc_info.value) diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py index f257b47404e..6b6b5d768dd 100644 --- a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py +++ b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py @@ -141,7 +141,7 @@ async def test_bedrock_apply_guardrail_api_failure(): mock_api_request.side_effect = Exception("API connection failed") # Test the apply_guardrail method should raise an exception - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Bedrock guardrail failed: API connection failed') as exc_info: await guardrail.apply_guardrail( inputs={"texts": ["This is a test message"]}, request_data={}, diff --git a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py index 714f3be6df9..2d845a445b5 100644 --- a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py +++ b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py @@ -1653,7 +1653,7 @@ async def test_afile_retrieve_raises_error_when_no_router_and_file_object_none() unified_file_id = "test-unified-file-id" - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='LiteLLM Managed File object with id=test-unified-file-id') as exc_info: await proxy_managed_files.afile_retrieve( file_id=unified_file_id, litellm_parent_otel_span=None, @@ -1719,7 +1719,7 @@ async def test_afile_retrieve_raises_error_for_non_managed_file(): # Mock get_unified_file_id to return None (file not found) proxy_managed_files.get_unified_file_id = AsyncMock(return_value=None) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='LiteLLM Managed File object with id=non-existent-file-id') as exc_info: await proxy_managed_files.afile_retrieve( file_id="non-existent-file-id", litellm_parent_otel_span=None, @@ -2027,7 +2027,7 @@ async def test_list_batches_from_managed_objects_table_provider_filter_raises_ex ) # Filtering by provider should raise Exception - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Filtering by 'provider' is not supported when using managed") as exc_info: await proxy_managed_files.list_user_batches( user_api_key_dict=UserAPIKeyAuth(user_id="test-user"), limit=10, @@ -2053,7 +2053,7 @@ async def test_list_batches_from_managed_objects_table_target_model_name_filter_ ) # Filtering by provider should raise Exception - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Filtering by 'target_model_names' is not supported when") as exc_info: await proxy_managed_files.list_user_batches( user_api_key_dict=UserAPIKeyAuth(user_id="test-user"), limit=10, diff --git a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py index bd6637ffcac..ed6735a7126 100644 --- a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py +++ b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py @@ -448,7 +448,7 @@ def test_check_team_project_limits_models_not_in_team(): models=["gpt-5.5", "claude-3"], # claude-3 not in team ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="not in team's allowed models\\. Team allowed models") as exc_info: _check_team_project_limits(team_object=team, data=data) assert "claude-3" in str(exc_info.value.detail) @@ -476,7 +476,7 @@ def test_check_team_project_limits_budget_exceeds_team(): max_budget=150.0, # exceeds team's 100.0 ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Project max_budget') as exc_info: _check_team_project_limits(team_object=team, data=data) assert "exceeds team's max_budget" in str(exc_info.value.detail) @@ -551,7 +551,7 @@ def test_check_team_project_limits_tpm_exceeds_team(): tpm_limit=20000, # exceeds team's 10000 ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Project tpm_limit') as exc_info: _check_team_project_limits(team_object=team, data=data) assert "exceeds team's tpm_limit" in str(exc_info.value.detail) @@ -577,7 +577,7 @@ def test_check_team_project_limits_negative_budget(): max_budget=-10.0, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='max_budget cannot be negative\\. Received') as exc_info: _check_team_project_limits(team_object=team, data=data) assert "cannot be negative" in str(exc_info.value.detail) @@ -604,7 +604,7 @@ def test_check_team_project_limits_soft_budget_gte_max(): soft_budget=100.0, # equal to max, should fail ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='must be strictly lower than max_budget') as exc_info: _check_team_project_limits(team_object=team, data=data) assert "must be strictly lower" in str(exc_info.value.detail) diff --git a/tests/guardrails_tests/test_dynamoai_guardrails.py b/tests/guardrails_tests/test_dynamoai_guardrails.py index 98f676a71d5..6f0ea00165b 100644 --- a/tests/guardrails_tests/test_dynamoai_guardrails.py +++ b/tests/guardrails_tests/test_dynamoai_guardrails.py @@ -61,7 +61,7 @@ async def test_dynamoai_blocks_content_with_block_action(): guardrail.should_run_guardrail = MagicMock(return_value=True) # Test that the guardrail raises ValueError for blocked content - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='violation\\(s\\) detected') as exc_info: await guardrail.async_pre_call_hook( data=request_data, user_api_key_dict=UserAPIKeyAuth(), diff --git a/tests/guardrails_tests/test_eu_ai_act_article5.py b/tests/guardrails_tests/test_eu_ai_act_article5.py index 0903e6c5416..f7384667481 100644 --- a/tests/guardrails_tests/test_eu_ai_act_article5.py +++ b/tests/guardrails_tests/test_eu_ai_act_article5.py @@ -211,7 +211,7 @@ class TestEUAIActArticle5ConditionalMatching: # Apply guardrail if expected == "BLOCK": # Should raise an exception or return modified response indicating block - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Content blocked: eu_ai_act_article') as exc_info: await content_filter_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, diff --git a/tests/guardrails_tests/test_eu_ai_act_french_3_scenarios.py b/tests/guardrails_tests/test_eu_ai_act_french_3_scenarios.py index 6b9774d9cde..221ca5aa6e6 100644 --- a/tests/guardrails_tests/test_eu_ai_act_french_3_scenarios.py +++ b/tests/guardrails_tests/test_eu_ai_act_french_3_scenarios.py @@ -83,7 +83,7 @@ class TestEUAIActFrench3Scenarios: print(f"{'='*70}\n") # Should raise an exception (blocked) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="prohibited_practices_fr conditional match 'concevoir \\+") as exc_info: await content_filter_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, @@ -123,7 +123,7 @@ class TestEUAIActFrench3Scenarios: print(f"{'='*70}\n") # Should raise an exception (blocked) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="prohibited_practices_fr conditional match 'créer \\+") as exc_info: await content_filter_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, @@ -194,7 +194,7 @@ class TestEUAIActFrench3Scenarios: print(f"{'='*70}\n") # Should raise an exception (blocked by conditional matching) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="prohibited_practices_fr conditional match 'développer \\+") as exc_info: await content_filter_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, @@ -278,7 +278,7 @@ class TestFrenchEdgeCases: request_data = {"messages": [{"role": "user", "content": sentence}]} # Should still block (no exception bypass) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="prohibited_practices_fr conditional match 'créer \\+ crédit") as exc_info: await content_filter_guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, diff --git a/tests/guardrails_tests/test_sg_mas_ai_guardrails.py b/tests/guardrails_tests/test_sg_mas_ai_guardrails.py index 668ee704692..e587d666a79 100644 --- a/tests/guardrails_tests/test_sg_mas_ai_guardrails.py +++ b/tests/guardrails_tests/test_sg_mas_ai_guardrails.py @@ -55,7 +55,7 @@ def _make_guardrail(yaml_filename: str, category_name: str) -> ContentFilterGuar async def _expect_block(guardrail: ContentFilterGuardrail, sentence: str, reason: str): request_data = {"messages": [{"role": "user", "content": sentence}]} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Content blocked: sg_mas_') as exc_info: await guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, diff --git a/tests/guardrails_tests/test_sg_pdpa_guardrails.py b/tests/guardrails_tests/test_sg_pdpa_guardrails.py index fd7133bc745..42c3a15f9f6 100644 --- a/tests/guardrails_tests/test_sg_pdpa_guardrails.py +++ b/tests/guardrails_tests/test_sg_pdpa_guardrails.py @@ -62,7 +62,7 @@ def _make_guardrail(yaml_filename: str, category_name: str) -> ContentFilterGuar async def _expect_block(guardrail: ContentFilterGuardrail, sentence: str, reason: str): """Assert that the guardrail BLOCKS the sentence.""" request_data = {"messages": [{"role": "user", "content": sentence}]} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Content blocked: sg_pdpa_') as exc_info: await guardrail.apply_guardrail( inputs={"texts": [sentence]}, request_data=request_data, diff --git a/tests/litellm_utils_tests/test_hashicorp.py b/tests/litellm_utils_tests/test_hashicorp.py index 9aff7ddc10e..fa39a045227 100644 --- a/tests/litellm_utils_tests/test_hashicorp.py +++ b/tests/litellm_utils_tests/test_hashicorp.py @@ -432,7 +432,7 @@ def test_hashicorp_get_url_rejects_path_traversal(monkeypatch, malicious_secret_ monkeypatch.setenv("HCP_VAULT_TOKEN", "test-token-for-get-url-only") manager = HashicorpSecretManager() - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Invalid secret_name'): manager.get_url(malicious_secret_name) diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py index 2b539b97c9b..3c73224d7a1 100644 --- a/tests/litellm_utils_tests/test_utils.py +++ b/tests/litellm_utils_tests/test_utils.py @@ -2147,7 +2147,7 @@ def test_validate_user_messages_invalid_content_type(): messages = [{"content": [{"type": "invalid_type", "text": "Hello"}]}] - with pytest.raises(Exception) as e: + with pytest.raises(Exception, match='Please ensure all messages are valid OpenAI chat completion') as e: validate_chat_completion_user_messages(messages) assert "Invalid message" in str(e) diff --git a/tests/litellm_utils_tests/test_validate_tool_choice.py b/tests/litellm_utils_tests/test_validate_tool_choice.py index 0e6294a7cd4..07f8c9ed8f4 100644 --- a/tests/litellm_utils_tests/test_validate_tool_choice.py +++ b/tests/litellm_utils_tests/test_validate_tool_choice.py @@ -37,27 +37,27 @@ def test_validate_tool_choice_cursor_format(): def test_validate_tool_choice_invalid_dict(): """Test that invalid dict formats raise exceptions.""" # Missing both type and function - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Invalid tool choice, tool_choice=\\{\\}\\. Please ensure') as exc_info: validate_chat_completion_tool_choice({}) assert "Invalid tool choice" in str(exc_info.value) # Invalid type value - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Invalid tool choice, tool_choice=\\{'type': 'invalid'\\}\\.") as exc_info: validate_chat_completion_tool_choice({"type": "invalid"}) assert "Invalid tool choice" in str(exc_info.value) # Has type but missing function when type is "function" - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Invalid tool choice, tool_choice=\\{'type': 'function'\\}\\.") as exc_info: validate_chat_completion_tool_choice({"type": "function"}) assert "Invalid tool choice" in str(exc_info.value) def test_validate_tool_choice_invalid_type(): """Test that invalid types raise exceptions.""" - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="\\. Expecting str, or dict\\. Please ensure") as exc_info: validate_chat_completion_tool_choice(123) assert "Got=" in str(exc_info.value) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Invalid tool choice, tool_choice=\\[\\]\\. Got=\\.") as exc_info: validate_chat_completion_tool_choice([]) assert "Got=" in str(exc_info.value) diff --git a/tests/llm_responses_api_testing/test_responses_hooks.py b/tests/llm_responses_api_testing/test_responses_hooks.py index 2344a62de4d..66dbb29dba5 100644 --- a/tests/llm_responses_api_testing/test_responses_hooks.py +++ b/tests/llm_responses_api_testing/test_responses_hooks.py @@ -295,7 +295,7 @@ async def test_responses_streaming_failure_triggers_failure_handlers(): call_type=CallTypes.responses.value, ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="boom"): iterator._process_chunk('{"delta": "chunk"}') # allow failure callbacks to run diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 3303fafafb0..9534bc8de3c 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -1890,7 +1890,7 @@ def test_bedrock_completion_test_4(modify_params): ] assert transformed_messages == expected_messages else: - with pytest.raises(Exception) as e: + with pytest.raises(Exception, match=r"litellm\.modify_params") as e: litellm.completion(**data) assert "litellm.modify_params" in str(e.value) diff --git a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py index cc493cc5a28..8c7390d3d04 100644 --- a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py +++ b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py @@ -982,7 +982,7 @@ def test_convert_to_model_response_object_with_real_error(): }, } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception) as exc_info: # noqa: PT011 # message rides on .message, str() is empty convert_to_model_response_object( model_response_object=ModelResponse(), response_object=response_object, @@ -1243,7 +1243,7 @@ def test_convert_to_model_response_object_with_error_code_only(): }, } - with pytest.raises(Exception) as exc_info: # noqa: B017 # bare Exception raised, so status_code is the assertion + with pytest.raises(Exception) as exc_info: # noqa: B017, PT011 # bare Exception, empty message, so status_code is the assertion convert_to_model_response_object( model_response_object=ModelResponse(), response_object=response_object, @@ -1423,7 +1423,7 @@ def test_error_message_includes_function_args(): "choices": [{"index": 0}], } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='in convert_to_model_response_object') as exc_info: convert_to_model_response_object( model_response_object=ModelResponse(), response_object=response_object, diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index c3519fcb40f..1b4c8a82cf4 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -1845,7 +1845,7 @@ def test_parse_tool_call_arguments_malformed_json(): parse_tool_call_arguments, ) - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="Failed to parse tool call arguments for tool 'load_skill") as exc_info: parse_tool_call_arguments( '{"skill_name": "pptx', tool_name="load_skill", @@ -1877,7 +1877,7 @@ def test_convert_to_anthropic_tool_invoke_malformed_json(): } ] - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="Failed to parse tool call arguments for tool 'bad_tool") as exc_info: convert_to_anthropic_tool_invoke(tool_calls) error_msg = str(exc_info.value) @@ -2023,7 +2023,7 @@ def test_parse_tool_call_arguments_still_raises_for_unrepairable(): parse_tool_call_arguments, ) - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="Failed to parse tool call arguments for tool 'test_tool") as exc_info: parse_tool_call_arguments( '{"key": "unterminated', tool_name="test_tool", diff --git a/tests/llm_translation/test_triton.py b/tests/llm_translation/test_triton.py index 21887e8d848..f4a26360a6c 100644 --- a/tests/llm_translation/test_triton.py +++ b/tests/llm_translation/test_triton.py @@ -45,7 +45,7 @@ def test_split_embedding_by_shape_fails_with_shape_value_error(): "data": [1, 2, 3, 4, 5, 6], } ] - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Shape must be of length'): TritonEmbeddingConfig.split_embedding_by_shape( data[0]["data"], data[0]["shape"] ) diff --git a/tests/llm_translation/test_unit_test_bedrock_invoke.py b/tests/llm_translation/test_unit_test_bedrock_invoke.py index 14f08c759c5..39f02263f03 100644 --- a/tests/llm_translation/test_unit_test_bedrock_invoke.py +++ b/tests/llm_translation/test_unit_test_bedrock_invoke.py @@ -59,7 +59,7 @@ def test_transform_request_invalid_provider(bedrock_transformer): """Test request transformation with invalid provider""" messages = [{"role": "user", "content": "Hello"}] - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Bedrock Invoke HTTPX: Unknown provider=None') as exc_info: bedrock_transformer.transform_request( model="invalid.model", messages=messages, diff --git a/tests/local_testing/test_auth_utils.py b/tests/local_testing/test_auth_utils.py index 9aecb7e10e4..88e8c02a606 100644 --- a/tests/local_testing/test_auth_utils.py +++ b/tests/local_testing/test_auth_utils.py @@ -264,10 +264,6 @@ def test_get_end_user_id_from_request_body_backwards_compatibility(): ["gpt-3.5-turbo", "gpt-4o-mini-general-deployment"], ), ({"model": "gpt-3.5-turbo"}, "gpt-3.5-turbo"), - ( - {"model": "gpt-3.5-turbo, gpt-4o-mini-general-deployment"}, - ["gpt-3.5-turbo", "gpt-4o-mini-general-deployment"], - ), ], ) def test_get_model_from_request(request_data, expected_model): diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index edf847f4cef..8dd90cbfb37 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -1433,7 +1433,7 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model): sync_stream=sync_mode, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='litellm\\.BadRequestError: OpenAIException - Invalid value') as exc_info: await _call_with_bad_role() assert exc_info.value.code == "invalid_value" diff --git a/tests/local_testing/test_file_types.py b/tests/local_testing/test_file_types.py index db83ba0e74b..7fda81ebd45 100644 --- a/tests/local_testing/test_file_types.py +++ b/tests/local_testing/test_file_types.py @@ -23,13 +23,13 @@ class TestFileConsts: def test_get_file_extension_from_mime_type(self): assert get_file_extension_from_mime_type("audio/aac") == "aac" assert get_file_extension_from_mime_type("application/pdf") == "pdf" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Unknown extension for mime type: application'): get_file_extension_from_mime_type("application/unknown") def test_get_file_type_from_extension(self): assert get_file_type_from_extension("aac") == FileType.AAC assert get_file_type_from_extension("pdf") == FileType.PDF - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Unknown file type for extension: unknown'): get_file_type_from_extension("unknown") def test_get_file_extension_for_file_type(self): diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index 385be25fb07..cef05050ac9 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -134,7 +134,6 @@ def test_get_model_info_bedrock_region(): "ft:gpt-3.5-turbo:my-org:custom_suffix:id", "ft:gpt-4-0613:my-org:custom_suffix:id", "ft:davinci-002:my-org:custom_suffix:id", - "ft:gpt-4-0613:my-org:custom_suffix:id", "ft:babbage-002:my-org:custom_suffix:id", "gpt-35-turbo", "ada", diff --git a/tests/local_testing/test_router_budget_limiter.py b/tests/local_testing/test_router_budget_limiter.py index 48915137138..4ef99ec8c12 100644 --- a/tests/local_testing/test_router_budget_limiter.py +++ b/tests/local_testing/test_router_budget_limiter.py @@ -160,7 +160,7 @@ async def test_provider_budgets_e2e_test_expect_to_fail(): await asyncio.sleep(2.5) for _ in range(3): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Exceeded budget for provider") as exc_info: await router.acompletion( messages=[{"role": "user", "content": "Hello, how are you?"}], model="anthropic/claude-sonnet-4-5-20250929", @@ -594,7 +594,7 @@ async def test_deployment_budgets_e2e_test_expect_to_fail(): await asyncio.sleep(2.5) for _ in range(3): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Exceeded budget for deployment") as exc_info: await router.acompletion( messages=[{"role": "user", "content": "Hello, how are you?"}], model="openai/gpt-4o-mini", @@ -646,7 +646,7 @@ async def test_tag_budgets_e2e_test_expect_to_fail(): await asyncio.sleep(2.5) for _ in range(3): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match=f"Exceeded budget for tag='{TAG_NAME}'") as exc_info: await router.acompletion( messages=[{"role": "user", "content": "Hello, how are you?"}], model="openai/gpt-4o-mini", diff --git a/tests/local_testing/test_router_fallbacks.py b/tests/local_testing/test_router_fallbacks.py index 15c6c5fec59..86dec406332 100644 --- a/tests/local_testing/test_router_fallbacks.py +++ b/tests/local_testing/test_router_fallbacks.py @@ -1430,7 +1430,7 @@ async def test_router_fallbacks_default_and_model_specific_fallbacks(sync_mode): messages=[{"role": "user", "content": "Hey, how's it going?"}], ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='litellm\\.AuthenticationError: AuthenticationError') as exc_info: await _call_bad_model() assert isinstance( exc_info.value, litellm.AuthenticationError diff --git a/tests/logging_callback_tests/test_standard_logging_payload.py b/tests/logging_callback_tests/test_standard_logging_payload.py index d13cdf1337a..6a632c32fc2 100644 --- a/tests/logging_callback_tests/test_standard_logging_payload.py +++ b/tests/logging_callback_tests/test_standard_logging_payload.py @@ -293,7 +293,7 @@ def test_cleanup_timestamps(): assert all(isinstance(x, float) for x in result) # Test invalid input - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="start_time is required, got=invalid of type "): StandardLoggingPayloadSetup.cleanup_timestamps( "invalid", end_float, completion_float ) diff --git a/tests/multi_instance_e2e_tests/test_update_team_e2e.py b/tests/multi_instance_e2e_tests/test_update_team_e2e.py index dfbfbd310ee..13091fd3df6 100644 --- a/tests/multi_instance_e2e_tests/test_update_team_e2e.py +++ b/tests/multi_instance_e2e_tests/test_update_team_e2e.py @@ -143,7 +143,7 @@ async def test_team_blocking_behavior_multi_instance(): assert team_info_4001["blocked"] is True, "Team should be blocked after update" # 8. Make a chat completion request on port 4000 with a new prompt; expect it to be blocked. - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match="(?i)blocked") as excinfo: await chat_completion_on_port( session, key=key, @@ -157,7 +157,7 @@ async def test_team_blocking_behavior_multi_instance(): ), f"Expected error indicating team blocked, got: {error_msg}" # 9. Make a chat completion request on port 4000 with a new prompt; expect it to be blocked. - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match="(?i)blocked") as excinfo: await chat_completion_on_port( session, key=key, @@ -171,7 +171,7 @@ async def test_team_blocking_behavior_multi_instance(): ), f"Expected error indicating team blocked, got: {error_msg}" # 9. Repeat the chat completion request with another new prompt; expect it to be blocked. - with pytest.raises(Exception) as excinfo_second: + with pytest.raises(Exception, match="(?i)blocked") as excinfo_second: await chat_completion_on_port( session, key=key, diff --git a/tests/ocr_tests/test_ocr_azure_document_intelligence.py b/tests/ocr_tests/test_ocr_azure_document_intelligence.py index 5736bd797e3..e6a2e5e5735 100644 --- a/tests/ocr_tests/test_ocr_azure_document_intelligence.py +++ b/tests/ocr_tests/test_ocr_azure_document_intelligence.py @@ -101,7 +101,7 @@ class TestAzureDocumentIntelligencePagesParam: cfg.map_ocr_params({"pages": [True, False]}, {}, "prebuilt-layout") def test_map_ocr_params_unsupported_type_raises(self, cfg): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='based, Mistral-style\\) or a string like'): cfg.map_ocr_params({"pages": 5}, {}, "prebuilt-layout") def test_get_complete_url_appends_pages_query(self, cfg): diff --git a/tests/otel_tests/test_e2e_model_access.py b/tests/otel_tests/test_e2e_model_access.py index 5b5f2a89c8d..e5e93c0b179 100644 --- a/tests/otel_tests/test_e2e_model_access.py +++ b/tests/otel_tests/test_e2e_model_access.py @@ -3,6 +3,7 @@ import asyncio import aiohttp import json from httpx import AsyncClient +from openai import PermissionDeniedError from typing import Any, Optional, List, Literal @@ -134,7 +135,7 @@ async def test_model_access_update(): await mock_chat_completion(session=session, key=key, model="openai/gpt-5.5") # Should fail with gpt-5-mini - with pytest.raises(Exception) as exc_info: + with pytest.raises(PermissionDeniedError) as exc_info: await mock_chat_completion( session=session, key=key, model="openai/gpt-5-mini" ) @@ -157,7 +158,7 @@ async def test_model_access_update(): ) # Non-OpenAI model should still fail - with pytest.raises(Exception) as exc_info: + with pytest.raises(PermissionDeniedError) as exc_info: await mock_chat_completion( session=session, key=key, model="anthropic/claude-2" ) @@ -254,7 +255,7 @@ async def test_team_model_access_update(): await mock_chat_completion(session=session, key=key, model="openai/gpt-5.5") # Should fail with gpt-5-mini - with pytest.raises(Exception) as exc_info: + with pytest.raises(PermissionDeniedError) as exc_info: await mock_chat_completion( session=session, key=key, model="openai/gpt-5-mini" ) @@ -279,7 +280,7 @@ async def test_team_model_access_update(): ) # Non-OpenAI model should still fail - with pytest.raises(Exception) as exc_info: + with pytest.raises(PermissionDeniedError) as exc_info: await mock_chat_completion( session=session, key=key, model="anthropic/claude-2" ) diff --git a/tests/proxy_admin_ui_tests/test_key_management.py b/tests/proxy_admin_ui_tests/test_key_management.py index 4c5a045509a..7e8494b77fc 100644 --- a/tests/proxy_admin_ui_tests/test_key_management.py +++ b/tests/proxy_admin_ui_tests/test_key_management.py @@ -1340,6 +1340,6 @@ async def test_team_model_alias(prisma_client, requested_model, should_pass): }, "Expected model aliases to be present" else: # Verify the key fails with non-aliased models - with pytest.raises(Exception) as exc_info: + with pytest.raises(ProxyException) as exc_info: await user_api_key_auth(request=request, api_key=f"Bearer {generated_key}") assert exc_info.value.type == ProxyErrorTypes.key_model_access_denied diff --git a/tests/proxy_admin_ui_tests/test_role_based_access.py b/tests/proxy_admin_ui_tests/test_role_based_access.py index 9398428bd67..f9506fb694b 100644 --- a/tests/proxy_admin_ui_tests/test_role_based_access.py +++ b/tests/proxy_admin_ui_tests/test_role_based_access.py @@ -9,7 +9,7 @@ from litellm._uuid import uuid from datetime import datetime from dotenv import load_dotenv -from fastapi import Request +from fastapi import HTTPException, Request from fastapi.routing import APIRoute load_dotenv() @@ -530,7 +530,7 @@ async def test_user_role_permissions(prisma_client, route, user_role, expected_r print(f"Auth passed as expected for {route} with role {user_role}") else: # Should raise an error - with pytest.raises(Exception) as exc_info: + with pytest.raises((ProxyException, HTTPException)) as exc_info: await user_api_key_auth(request=request, api_key=bearer_token) print(f"Auth failed as expected for {route} with role {user_role}") print(f"Error message: {str(exc_info.value)}") diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/proxy_unit_tests/test_auth_checks.py index ffcbe472be7..ef3cbd0ae95 100644 --- a/tests/proxy_unit_tests/test_auth_checks.py +++ b/tests/proxy_unit_tests/test_auth_checks.py @@ -173,7 +173,7 @@ async def test_can_key_call_model(model, expect_to_work): if expect_to_work: await can_key_call_model(**args) else: - with pytest.raises(Exception) as e: + with pytest.raises(Exception, match='key not allowed to access model\\. This key can only access') as e: await can_key_call_model(**args) print(e) @@ -958,7 +958,7 @@ async def test_can_key_call_model_with_aliases(model, alias_map, expect_to_work) llm_router=router, ) else: - with pytest.raises(Exception) as e: + with pytest.raises(Exception, match='key not allowed to access model\\. This key can only access') as e: await can_key_call_model( model=model, llm_model_list=llm_model_list, diff --git a/tests/proxy_unit_tests/test_jwt.py b/tests/proxy_unit_tests/test_jwt.py index 4db47a1cde4..686d7021257 100644 --- a/tests/proxy_unit_tests/test_jwt.py +++ b/tests/proxy_unit_tests/test_jwt.py @@ -1583,7 +1583,7 @@ async def test_auth_jwt_mismatched_key_fails(monkeypatch): h = JWTHandler() with patch.object(h, "get_public_key", new=AsyncMock(return_value=rsa_jwk)): - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='Validation fails: Expecting a PEM-formatted key\\.') as exc: await h.auth_jwt(token) assert "Validation fails" in str(exc.value) @@ -1826,7 +1826,7 @@ async def test_multi_issuer_jwt_unknown_issuer_without_global_jwks_rejected( kid="issuer-key", ) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='Missing JWT Public Key URL from environment\\.') as exc: await jwt_handler.auth_jwt(token=token) assert "Missing JWT Public Key URL" in str(exc.value) @@ -1857,7 +1857,7 @@ async def test_multi_issuer_jwt_rejects_wrong_audience(monkeypatch): kid="issuer-key", ) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match="Validation fails: Audience doesn't match") as exc: await jwt_handler.auth_jwt(token=token) assert "Validation fails" in str(exc.value) @@ -1900,7 +1900,7 @@ async def test_multi_issuer_jwt_same_kid_does_not_cross_issuer_keys(monkeypatch) kid=shared_kid, ) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='Validation fails: Signature verification failed') as exc: await jwt_handler.auth_jwt(token=token) assert "Validation fails" in str(exc.value) @@ -1953,7 +1953,7 @@ def test_multi_issuer_jwt_requires_audience_unless_explicitly_disabled( issuer = "https://issuer.example.com" jwks_url = f"{issuer}/keys" - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='must configure audience or set') as exc: LiteLLM_JWTAuth( issuers=[ { diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 00ed6d13f63..de2a9282300 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -1026,7 +1026,7 @@ def test_enforced_params_check( from litellm.proxy.litellm_pre_call_utils import _enforced_params_check if expected_error: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='in request body\\. This is a required param'): _enforced_params_check( request_body=request_body, general_settings=general_settings, @@ -2626,7 +2626,7 @@ async def test_during_call_hook_parallel_execution_with_error(): try: litellm.callbacks = [FailingGuardrail()] - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Guardrail violation detected!') as exc_info: await proxy_logging.during_call_hook( data={ "model": "gpt-4", diff --git a/tests/proxy_unit_tests/test_update_spend.py b/tests/proxy_unit_tests/test_update_spend.py index 0d1d6dcf3c6..6b8973fbad2 100644 --- a/tests/proxy_unit_tests/test_update_spend.py +++ b/tests/proxy_unit_tests/test_update_spend.py @@ -166,7 +166,7 @@ async def test_update_spend_logs_non_connection_error(): prisma_client.db.litellm_spendlogs.create_many = create_many_mock # Execute and verify it raises immediately without retrying - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Unexpected database error') as exc_info: await update_spend(prisma_client, None, proxy_logging_obj) # Verify error message diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 1fef0f01df8..f81578dbd99 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -90,7 +90,7 @@ def test_routing_strategy_init_invalid_strategy(model_list): router = Router(model_list=model_list) # Test common mistake: "simple" instead of "simple-shuffle" - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="usage-based-routing', 'provider-budget-routing'\\]\\. Check") as exc_info: router.routing_strategy_init( routing_strategy="simple", routing_strategy_args={} ) @@ -106,7 +106,7 @@ def test_routing_strategy_init_invalid_strategy(model_list): assert "Router SDK" in error_msg # Test completely invalid strategy - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="usage-based-routing', 'provider-budget-routing'\\]\\. Check") as exc_info: router.routing_strategy_init( routing_strategy="not-a-real-strategy", routing_strategy_args={} ) diff --git a/tests/store_model_in_db_tests/test_mcp_servers.py b/tests/store_model_in_db_tests/test_mcp_servers.py index e9c26221580..735d5d71ad3 100644 --- a/tests/store_model_in_db_tests/test_mcp_servers.py +++ b/tests/store_model_in_db_tests/test_mcp_servers.py @@ -471,7 +471,7 @@ def test_validate_mcp_server_name_direct(): validate_mcp_server_name("valid name") # Test that invalid names with hyphens raise exceptions - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Server name cannot contain '-'\\. Use an alternative") as exc_info: validate_mcp_server_name("invalid-name") assert "cannot contain" in str(exc_info.value) diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py index 40439a78a49..5fe4b217e4f 100644 --- a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py +++ b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py @@ -104,7 +104,7 @@ async def test_send_email_missing_api_key(): try: logger = SendGridEmailLogger() - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='SENDGRID_API_KEY is not set'): await logger.send_email( from_email="test@example.com", to_email=["recipient@example.com"], diff --git a/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py b/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py index 6cc31f991a3..fcd03e77aa2 100644 --- a/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py +++ b/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py @@ -471,7 +471,7 @@ async def test_afile_content_error_reports_unified_id_not_provider_uri(): mock_router.get_deployment_credentials_with_provider = MagicMock(return_value=None) mock_router.afile_content = AsyncMock(side_effect=Exception("deployment failed")) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='LiteLLM Managed File object with') as exc_info: await managed_files.afile_content( file_id=unified_file_id, litellm_parent_otel_span=None, diff --git a/tests/test_litellm/integrations/cloudzero/test_cloudzero_database.py b/tests/test_litellm/integrations/cloudzero/test_cloudzero_database.py index 89a5028011c..7f930f90247 100644 --- a/tests/test_litellm/integrations/cloudzero/test_cloudzero_database.py +++ b/tests/test_litellm/integrations/cloudzero/test_cloudzero_database.py @@ -51,7 +51,7 @@ async def test_get_usage_data_rejects_invalid_limit(monkeypatch: pytest.MonkeyPa """limit must coerce to int or raise ValueError before hitting the DB.""" db, query_mock = _setup_db(monkeypatch, []) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='limit must be an integer'): await db.get_usage_data(limit="invalid") assert query_mock.await_count == 0 diff --git a/tests/test_litellm/integrations/cloudzero/test_cz_stream_api.py b/tests/test_litellm/integrations/cloudzero/test_cz_stream_api.py index 440ce39e021..a715116e5ee 100644 --- a/tests/test_litellm/integrations/cloudzero/test_cz_stream_api.py +++ b/tests/test_litellm/integrations/cloudzero/test_cz_stream_api.py @@ -108,7 +108,7 @@ class TestCloudZeroStreamer: """Test _parse_and_convert_timestamp method with invalid timestamp.""" streamer = CloudZeroStreamer("test-key", "test-connection") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Could not parse timestamp 'invalid-timestamp': Invalid"): streamer._parse_and_convert_timestamp("invalid-timestamp") def test_prepare_batch_payload(self): diff --git a/tests/test_litellm/integrations/focus/test_focus_database.py b/tests/test_litellm/integrations/focus/test_focus_database.py index d77af2dd170..5c13665f1f1 100644 --- a/tests/test_litellm/integrations/focus/test_focus_database.py +++ b/tests/test_litellm/integrations/focus/test_focus_database.py @@ -68,7 +68,7 @@ async def test_should_accept_string_timestamps(monkeypatch: pytest.MonkeyPatch): async def test_should_reject_invalid_limit(monkeypatch: pytest.MonkeyPatch): db, query_mock = _setup_db(monkeypatch, []) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='limit must be an integer'): await db.get_usage_data(limit="invalid") assert query_mock.await_count == 0 diff --git a/tests/test_litellm/integrations/focus/test_s3_destination.py b/tests/test_litellm/integrations/focus/test_s3_destination.py index f915b2c56a3..8e54b561f82 100644 --- a/tests/test_litellm/integrations/focus/test_s3_destination.py +++ b/tests/test_litellm/integrations/focus/test_s3_destination.py @@ -20,7 +20,7 @@ def _window(freq: str = "hourly", hour: int = 5) -> FocusTimeWindow: def test_should_require_bucket_name(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='bucket_name must be provided for S'): FocusS3Destination(prefix="focus", config={}) diff --git a/tests/test_litellm/integrations/gitlab/test_gitlab_client.py b/tests/test_litellm/integrations/gitlab/test_gitlab_client.py index 4556950cd3e..529868ca06a 100644 --- a/tests/test_litellm/integrations/gitlab/test_gitlab_client.py +++ b/tests/test_litellm/integrations/gitlab/test_gitlab_client.py @@ -95,9 +95,9 @@ def enc_project(p): # how client encodes project in urls # Constructor / config tests # ----------------------------- def test_init_requires_project_and_token(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='project and access_token are required'): GitLabClient({"project": "p"}) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='project and access_token are required'): GitLabClient({"access_token": "t"}) @@ -127,7 +127,7 @@ def test_set_ref_updates_effective_ref(): c = make_client(branch="main") c.set_ref("feature/x") assert c.ref == "feature/x" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='ref must be a non-empty string'): c.set_ref("") @@ -193,12 +193,12 @@ def test_get_file_content_permission_errors_are_mapped(): raw_url = f"https://gitlab.example.com/api/v4/projects/{enc_project('group/sub/repo')}/repository/files/secure%2Ffile.prompt/raw?ref=main" # raise_for_status will be called, so return 403 response (not an exception from transport) c.http_handler.routes[raw_url] = FakeResponse(status_code=403) - with pytest.raises(Exception) as ei: + with pytest.raises(Exception, match="Check your GitLab permissions for project 'group") as ei: c.get_file_content("secure/file.prompt") assert "Access denied" in str(ei.value) c.http_handler.routes[raw_url] = FakeResponse(status_code=401) - with pytest.raises(Exception) as ei2: + with pytest.raises(Exception, match='Authentication failed\\. Check your GitLab token and') as ei2: c.get_file_content("secure/file.prompt") assert "Authentication failed" in str(ei2.value) diff --git a/tests/test_litellm/integrations/levo/test_levo.py b/tests/test_litellm/integrations/levo/test_levo.py index 98b0327dbf2..903be644671 100644 --- a/tests/test_litellm/integrations/levo/test_levo.py +++ b/tests/test_litellm/integrations/levo/test_levo.py @@ -198,7 +198,7 @@ class TestLevoIntegration(unittest.TestCase): """Test health check returns unhealthy status when required vars are missing.""" # Try to create logger without required env vars # This should fail during config, but we can test health check logic - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='LEVOAI_API_KEY environment variable is required for Levo'): LevoLogger.get_levo_config() @patch.dict( diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py b/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py index e1b8e4b5721..b810ffdc6be 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py @@ -554,7 +554,7 @@ def test_token_type_rejected_from_either_list(attributes, monkeypatch): recorder rather than silently ignored, so the misconfig is caught at all.""" recorder = _recorder(monkeypatch, attributes) kwargs, response_obj, start, end = _build_call() - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='otel\\.attributes: gen_ai\\.token\\.type is a structural') as exc_info: recorder.record(kwargs, response_obj, start, end) # The dedicated discriminator guard, not the generic unknown-name path: assert # the specific reason so dropping that guard (and falling through to "unknown diff --git a/tests/test_litellm/integrations/test_langfuse.py b/tests/test_litellm/integrations/test_langfuse.py index 6e57a36c5b6..3c7dd51bff8 100644 --- a/tests/test_litellm/integrations/test_langfuse.py +++ b/tests/test_litellm/integrations/test_langfuse.py @@ -1163,7 +1163,7 @@ def test_max_langfuse_clients_limit(): assert litellm.initialized_langfuse_clients == 2 # Third client should fail with exception - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Max langfuse clients reached') as exc_info: logger3 = LangFuseLogger( langfuse_public_key="test_key_3", langfuse_secret="test_secret_3", diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index fffbc884782..08d8c17cc2e 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1169,7 +1169,7 @@ def test_bedrock_image_processor_content_type_fallback_failure(): # Test with URL without recognizable extension image_url = "https://example.com/unknown-file" - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match='Unable to determine content type from URL: https') as excinfo: BedrockImageProcessor._post_call_image_processing(mock_response, image_url) assert "Unable to determine content type" in str(excinfo.value) diff --git a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py index 0dca4f3a1b1..956f86a9292 100644 --- a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py +++ b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py @@ -110,7 +110,7 @@ def test_top_level_kwargs_overrides_metadata_slots(): def test_env_reference_at_top_level_raises_with_guidance(): kwargs = {"langfuse_public_key": "os.environ/LANGFUSE_PUBLIC_KEY"} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="Callback param 'langfuse_public_key' \\(from request body\\)") as exc_info: initialize_standard_callback_dynamic_params(kwargs) message = str(exc_info.value) @@ -127,7 +127,7 @@ def test_env_reference_in_metadata_raises_with_guidance(): } } - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="Callback param 'langsmith_api_key' \\(from metadata\\) contains") as exc_info: initialize_standard_callback_dynamic_params(kwargs) message = str(exc_info.value) diff --git a/tests/test_litellm/litellm_core_utils/test_llm_judge.py b/tests/test_litellm/litellm_core_utils/test_llm_judge.py index 5c092caa7c3..a0a2311914b 100644 --- a/tests/test_litellm/litellm_core_utils/test_llm_judge.py +++ b/tests/test_litellm/litellm_core_utils/test_llm_judge.py @@ -27,7 +27,7 @@ def test_parse_json_verdict_tolerates_fences_and_prose(raw, expected): def test_parse_json_verdict_rejects_non_object(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='judge response is not a JSON object'): parse_json_verdict('["not", "an", "object"]') with pytest.raises((json.JSONDecodeError, ValueError)): parse_json_verdict("no json here at all") diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 456d4db4afe..fbdfcac1adc 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -982,7 +982,7 @@ async def test_bedrock_validation_error_raises_directly(logging_obj: Logging): make_call=_raise_400, ) - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match='litellm\\.BadRequestError: BedrockException') as excinfo: await response.__anext__() assert not isinstance(excinfo.value, MidStreamFallbackError) assert getattr(excinfo.value, "status_code", None) == 400 @@ -2722,7 +2722,7 @@ def test_dispatch_text_completion_codestral_requires_string( is a programming error and must surface loudly.""" initialized_custom_stream_wrapper.custom_llm_provider = "text-completion-codestral" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="chunk is not a string: \\{'not': 'a string'\\}"): _run_dispatch(initialized_custom_stream_wrapper, {"not": "a string"}) diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 3c33ee13c3f..eec4b307c87 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -763,24 +763,6 @@ class TestTokenizerSelection(unittest.TestCase): ], } ], - [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "These are some sample images from a movie. Based on these images, what do you think the tone of the movie is?", - }, - { - "type": "text", - "image_url": { - "url": "https://gratisography.com/wp-content/uploads/2024/11/gratisography-augmented-reality-800x525.jpg", - "detail": "high", - }, - }, - ], - } - ], ], ) def test_bad_input_token_counter(model, messages): @@ -1174,7 +1156,7 @@ def test_count_content_list_rejects_unknown_type(): """ from litellm.litellm_core_utils.token_counter import _count_content_list - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Error getting number of tokens from content list: Invalid') as exc_info: _count_content_list( count_function=len, content_list=[{"type": "totally_unknown_block"}], diff --git a/tests/test_litellm/litellm_core_utils/test_url_utils.py b/tests/test_litellm/litellm_core_utils/test_url_utils.py index cef09f3f2b0..751b548adcd 100644 --- a/tests/test_litellm/litellm_core_utils/test_url_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_url_utils.py @@ -100,12 +100,12 @@ class TestEncodeUrlPathSegment: @pytest.mark.parametrize("value", ["", ".", "..", None]) def test_rejects_empty_and_dot_segments(self, value): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="resource_id (is required|cannot be a dot path segment)"): encode_url_path_segment(value, field_name="resource_id") @pytest.mark.parametrize("value", ["../model", "model/../other", "/model"]) def test_rejects_dot_segments_in_multi_segment_paths(self, value): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="model (is required|cannot be a dot path segment)"): encode_url_path_segments(value, field_name="model") diff --git a/tests/test_litellm/llms/aiml/image_generation/test_aiml_image_generation_transformation.py b/tests/test_litellm/llms/aiml/image_generation/test_aiml_image_generation_transformation.py index 7485f2121df..dd74379a883 100644 --- a/tests/test_litellm/llms/aiml/image_generation/test_aiml_image_generation_transformation.py +++ b/tests/test_litellm/llms/aiml/image_generation/test_aiml_image_generation_transformation.py @@ -113,7 +113,7 @@ def test_flux_style_request_still_remaps_to_legacy_fields(): def test_openai_style_unsupported_param_raises_without_drop_params(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Supported parameters are'): AimlImageGenerationConfig().map_openai_params( non_default_params={"image_size": {"width": 1024, "height": 1024}}, optional_params={}, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py index 654b0097546..f3cb2956aeb 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py @@ -61,7 +61,7 @@ def test_anthropic_messages_handler_skips_the_gateway_on_recursion(): "litellm.llms.anthropic.experimental_pass_through.messages.mcp_handler.anthropic_messages_with_mcp", new=AsyncMock(return_value={"routed": True}), ) as routed: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='anthropic_messages_handler is not implemented for sync calls'): anthropic_messages_handler( max_tokens=100, messages=[{"role": "user", "content": "hi"}], @@ -80,7 +80,7 @@ def test_anthropic_messages_handler_leaves_native_tools_alone(): "litellm.llms.anthropic.experimental_pass_through.messages.mcp_handler.anthropic_messages_with_mcp", new=AsyncMock(return_value={"routed": True}), ) as routed: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='anthropic_messages_handler is not implemented for sync calls'): anthropic_messages_handler( max_tokens=100, messages=[{"role": "user", "content": "hi"}], diff --git a/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py b/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py index ffabce6e00c..602cbf68f3f 100644 --- a/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py +++ b/tests/test_litellm/llms/azure_ai/rerank/test_azure_ai_rerank_transformation.py @@ -16,7 +16,7 @@ class TestAzureAIRerankConfigGetCompleteUrl: self.model = "azure_ai/cohere-rerank-v3-english" def test_api_base_required(self): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Azure AI API Base is required\\. api_base=None\\. Set in') as exc_info: self.config.get_complete_url(api_base=None, model=self.model) assert "api_base=None" in str(exc_info.value) @@ -31,7 +31,7 @@ class TestAzureAIRerankConfigGetCompleteUrl: ], ) def test_api_base_requires_scheme(self, api_base): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Azure AI API Base must be an absolute URL including scheme') as exc_info: self.config.get_complete_url(api_base=api_base, model=self.model) error_message = str(exc_info.value).lower() diff --git a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py index cfe9930e76e..b9f8283b78e 100644 --- a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py +++ b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py @@ -1944,7 +1944,7 @@ def test_role_assumption_access_denied_raises_when_different_role(): with patch.object( base_aws_llm, "_is_already_running_as_role", return_value=False ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='An error occurred \\(AccessDenied\\) when calling the') as exc_info: base_aws_llm._auth_with_aws_role( aws_access_key_id=None, aws_secret_access_key=None, @@ -1969,7 +1969,7 @@ def test_role_assumption_non_access_denied_error_propagated(): ) with patch("boto3.client", return_value=mock_sts_client): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='An error occurred \\(MalformedPolicyDocument\\) when calling') as exc_info: base_aws_llm._auth_with_aws_role( aws_access_key_id=None, aws_secret_access_key=None, diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index 8281f3387d9..28c8e5c7ed6 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -109,7 +109,7 @@ class TestBedrockMantleResponsesURL: monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) monkeypatch.delenv("AWS_REGION", raising=False) cfg = BedrockMantleResponsesAPIConfig() - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="api\\.aws\\.attacker\\.example/'\\. Region names must contain only"): cfg.get_complete_url( api_base=None, litellm_params={ @@ -1418,7 +1418,7 @@ class TestBedrockMantleResponsesSigV4: signer.get_credentials = MagicMock(side_effect=NoCredentialsError()) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, @@ -1448,7 +1448,7 @@ class TestBedrockMantleResponsesSigV4: signer.get_credentials = MagicMock(side_effect=cred_error) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index 275fb460b9f..07910b0b56f 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -107,7 +107,7 @@ class TestBedrockMantleConfig: monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) monkeypatch.delenv("AWS_REGION", raising=False) cfg = BedrockMantleChatConfig() - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="api\\.aws\\.attacker\\.example/'\\. Region names must contain only"): cfg._get_openai_compatible_provider_info( None, None, @@ -416,7 +416,7 @@ class TestBedrockMantleChatAuth: signer.get_credentials = MagicMock(side_effect=NoCredentialsError()) cfg = BedrockMantleChatConfig(aws_signer=signer) - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, diff --git a/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py b/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py index e2421437720..94b8c51dd52 100644 --- a/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py +++ b/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py @@ -38,7 +38,7 @@ class TestBytezChatConfig: config = BytezChatConfig() headers = {} - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match='Missing api_key, make sure you pass in your api key') as excinfo: config.validate_environment( headers=headers, model=TEST_MODEL, diff --git a/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank_transformation.py b/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank_transformation.py index a5411078cf7..ae3c166e7aa 100644 --- a/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank_transformation.py +++ b/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank_transformation.py @@ -258,7 +258,7 @@ class TestDeepinfraRerankTransform: status_code = 401 headers = {"content-type": "application/json"} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Authentication failed') as exc_info: self.config.get_error_class(error_message, status_code, headers) # The method should raise a BaseLLMException @@ -271,7 +271,7 @@ class TestDeepinfraRerankTransform: status_code = 404 headers = {"content-type": "application/json"} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Model not found') as exc_info: self.config.get_error_class(error_message, status_code, headers) # Should extract the nested error message @@ -284,7 +284,7 @@ class TestDeepinfraRerankTransform: status_code = 503 headers = {"content-type": "application/json"} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Service unavailable') as exc_info: self.config.get_error_class(error_message, status_code, headers) # Should extract the string detail @@ -296,7 +296,7 @@ class TestDeepinfraRerankTransform: status_code = 500 headers = {"content-type": "application/json"} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Invalid JSON error message') as exc_info: self.config.get_error_class(error_message, status_code, headers) # Should use the original error message when JSON parsing fails diff --git a/tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py b/tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py index 593593bfa73..c0f74eff51b 100644 --- a/tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py +++ b/tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py @@ -113,7 +113,7 @@ def test_response_format_is_ignored(): def test_unsupported_param_raises_without_drop_params(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Supported parameters are \\['n', 'response_format', 'size'\\]\\."): FalAINanoBananaConfig().map_openai_params( non_default_params={"style": "vivid"}, optional_params={}, diff --git a/tests/test_litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py b/tests/test_litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py index 4dc467575a0..bf40abd7016 100644 --- a/tests/test_litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py +++ b/tests/test_litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py @@ -44,7 +44,7 @@ class TestFeatherlessAIConfig: """Test error handling when API key is missing""" config = FeatherlessAIConfig() - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match='Missing Featherless AI API Key') as excinfo: config.validate_environment( headers={}, model="featherless-ai/Qwerky-72B", @@ -112,7 +112,7 @@ class TestFeatherlessAIConfig: "tool_choice": {"type": "function", "function": {"name": "get_weather"}} } optional_params = {} - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match="litellm\\.UnsupportedParamsError: Featherless AI doesn't") as excinfo: config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, @@ -138,7 +138,7 @@ class TestFeatherlessAIConfig: assert "tools" not in result # Test with tools and drop_params=False - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match="litellm\\.UnsupportedParamsError: Featherless AI doesn't") as excinfo: config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, diff --git a/tests/test_litellm/llms/fireworks_ai/rerank/test_fireworks_ai_rerank_transformation.py b/tests/test_litellm/llms/fireworks_ai/rerank/test_fireworks_ai_rerank_transformation.py index 30bf5860dee..521ea4f8263 100644 --- a/tests/test_litellm/llms/fireworks_ai/rerank/test_fireworks_ai_rerank_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/rerank/test_fireworks_ai_rerank_transformation.py @@ -301,7 +301,7 @@ class TestFireworksAIRerankTransform: mock_logging = MagicMock() model_response = RerankResponse() - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Failed to parse response: Invalid JSON: line') as exc_info: self.config.transform_rerank_response( model=self.model, raw_response=mock_response, diff --git a/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py b/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py index 9b57e1991de..bd9b7006e58 100644 --- a/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py +++ b/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py @@ -244,7 +244,7 @@ class TestGeminiImageEditTransformation: def test_transform_image_edit_request_without_image_raises(self) -> None: optional_params = {} - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Gemini image edit requires at least one image\\.'): self.config.transform_image_edit_request( model=self.model, prompt=self.prompt, diff --git a/tests/test_litellm/llms/gemini/test_gemini_client_setup.py b/tests/test_litellm/llms/gemini/test_gemini_client_setup.py index 51c6fedf5b8..48b010aca48 100644 --- a/tests/test_litellm/llms/gemini/test_gemini_client_setup.py +++ b/tests/test_litellm/llms/gemini/test_gemini_client_setup.py @@ -28,7 +28,7 @@ def test_gemini_completion_no_api_key(): del os.environ[key] # Test without mock_response to ensure actual API key validation - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='in _complete_vertex_ai_beta') as exc_info: completion( model="gemini/gemini-1.5-flash", messages=[{"role": "user", "content": "Test message"}], @@ -60,7 +60,7 @@ def test_gemini_completion_no_api_key_with_mock(): with patch("litellm.get_secret") as mock_get_secret: mock_get_secret.return_value = None - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='in _complete_vertex_ai_beta') as exc_info: completion( model="gemini/gemini-1.5-flash", messages=[{"role": "user", "content": "Test message"}], diff --git a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py index 6425e815db0..e6e6aa946d5 100644 --- a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py @@ -109,7 +109,7 @@ class TestHostedVLLMRerankTransform: ) assert url2 == "https://api.example.com/rerank" # Raises if api_base is None - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='api_base must be provided for Hosted VLLM rerank'): self.config.get_complete_url(None, self.model) def test_transform_response(self): diff --git a/tests/test_litellm/llms/langflow/chat/test_langflow_chat_transformation.py b/tests/test_litellm/llms/langflow/chat/test_langflow_chat_transformation.py index b3c4e0f1858..0c241add77b 100644 --- a/tests/test_litellm/llms/langflow/chat/test_langflow_chat_transformation.py +++ b/tests/test_litellm/llms/langflow/chat/test_langflow_chat_transformation.py @@ -46,7 +46,7 @@ def test_langflow_config_get_complete_url(): def test_langflow_config_get_complete_url_requires_api_base(): config = LangFlowConfig() - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='api_base is required for LangFlow\\. Set it via'): config.get_complete_url( api_base=None, api_key=None, diff --git a/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py b/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py index 7f00f53c451..fbcec3d4d2e 100644 --- a/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py +++ b/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py @@ -154,7 +154,7 @@ class TestModelScopeImageGenerationTransformation: mock_get_secret.return_value = None headers = {} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='MODELSCOPE_API_KEY is not set\\. Please set it via') as exc_info: self.config.validate_environment( headers=headers, model=self.model, @@ -367,7 +367,7 @@ class TestModelScopeImageGenerationTransformation: model_response = ImageResponse(data=[]) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='litellm\\.BadRequestError: ModelScope error: Invalid prompt') as exc_info: self.config.transform_image_generation_response( model=self.model, raw_response=mock_response, @@ -393,7 +393,7 @@ class TestModelScopeImageGenerationTransformation: model_response = ImageResponse(data=[]) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='litellm\\.InternalServerError: Error parsing ModelScope') as exc_info: self.config.transform_image_generation_response( model=self.model, raw_response=mock_response, diff --git a/tests/test_litellm/llms/novita/chat/test_novita_chat_transformation.py b/tests/test_litellm/llms/novita/chat/test_novita_chat_transformation.py index 7a00b361252..ade5e4176e8 100644 --- a/tests/test_litellm/llms/novita/chat/test_novita_chat_transformation.py +++ b/tests/test_litellm/llms/novita/chat/test_novita_chat_transformation.py @@ -47,7 +47,7 @@ class TestNovitaConfig: """Test error handling when API key is missing""" config = NovitaConfig() - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match='Missing Novita AI API Key - A call is being made to novita') as excinfo: config.validate_environment( headers={}, model="novita/meta-llama/llama-3.3-70b-instruct", diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py index 8be0780d86f..5aa96a66d2d 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -98,7 +98,7 @@ class TestOCIChatConfig: config = OCIChatConfig() headers = {} - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match='Missing required parameters: oci_user, oci_fingerprint') as excinfo: config.validate_environment( headers=headers, model=TEST_MODEL, @@ -272,7 +272,7 @@ class TestOCIChatConfig: "oci_serving_mode": "INVALID_MODE", } - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match="kwarg `oci_serving_mode` must be either 'ON_DEMAND' or") as excinfo: config.transform_request( model=TEST_MODEL_NAME, messages=TEST_MESSAGES, # type: ignore @@ -892,7 +892,7 @@ class TestOCISignerSupport: optional_params = {"oci_signer": MockSigner(), "method": "INVALID"} - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match='Unsupported HTTP method: INVALID') as excinfo: config.sign_request( headers={}, optional_params=optional_params, @@ -1604,7 +1604,7 @@ class TestOCIKeyNormalization: # We can't fully test signing without a real key, but we can verify # the error message indicates the key was processed (not a type error) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='why-can-t-i-import-my-pem-file for more details\\.') as exc_info: sign_with_manual_credentials( headers={}, optional_params=optional_params, @@ -1630,7 +1630,7 @@ class TestOCIKeyNormalization: "oci_key": crlf_pem, } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='why-can-t-i-import-my-pem-file for more details\\.') as exc_info: sign_with_manual_credentials( headers={}, optional_params=optional_params, @@ -1692,7 +1692,7 @@ class TestOCIValidateEnvironment: def test_missing_required_credentials_raises_error(self, config): """Test that missing required credentials raise an error.""" - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Missing required parameters: oci_user, oci_fingerprint') as exc_info: config.validate_environment( headers={}, model="oci/xai.grok-3", @@ -1875,7 +1875,7 @@ class TestOCIImageUrlTransformation: } ] - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Prop `image_url` must be a string or an object with a `url`') as exc_info: adapt_messages_to_generic_oci_standard(messages) assert "image_url" in str(exc_info.value) @@ -1899,7 +1899,7 @@ class TestOCIImageUrlTransformation: } ] - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Prop `image_url` must be a string or an object with a `url`') as exc_info: adapt_messages_to_generic_oci_standard(messages) assert "image_url" in str(exc_info.value) diff --git a/tests/test_litellm/llms/pg_vector/vector_stores/test_pg_vector_transformation.py b/tests/test_litellm/llms/pg_vector/vector_stores/test_pg_vector_transformation.py index 56953a574d6..1d44b2bc278 100644 --- a/tests/test_litellm/llms/pg_vector/vector_stores/test_pg_vector_transformation.py +++ b/tests/test_litellm/llms/pg_vector/vector_stores/test_pg_vector_transformation.py @@ -42,7 +42,7 @@ class TestPGVectorStoreConfig: litellm_params = GenericLiteLLMParams() headers = {} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='PG Vector API key is required\\. Set PG_VECTOR_API_KEY') as exc_info: config.validate_environment(headers, litellm_params) assert "PG Vector API key is required" in str(exc_info.value) @@ -84,7 +84,7 @@ class TestPGVectorStoreConfig: config = PGVectorStoreConfig() litellm_params = {} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='PG Vector API base URL is required\\. Set') as exc_info: config.get_complete_url(None, litellm_params) assert "PG Vector API base URL is required" in str(exc_info.value) diff --git a/tests/test_litellm/llms/recraft/image_edit/test_recraft_image_edit_transformation.py b/tests/test_litellm/llms/recraft/image_edit/test_recraft_image_edit_transformation.py index 0acabd05805..47811321133 100644 --- a/tests/test_litellm/llms/recraft/image_edit/test_recraft_image_edit_transformation.py +++ b/tests/test_litellm/llms/recraft/image_edit/test_recraft_image_edit_transformation.py @@ -167,7 +167,7 @@ class TestRecraftImageEditTransformation: mock_response.status_code = 500 mock_response.headers = {} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Error transforming image edit response: Invalid JSON: line') as exc_info: self.config.transform_image_edit_response( model=self.model, raw_response=mock_response, diff --git a/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py b/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py index 70311201969..ccc72dde7b8 100644 --- a/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py +++ b/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py @@ -64,7 +64,7 @@ class TestRecraftImageGenerationTransformation: non_default_params = {"n": 2, "unsupported_param": "value"} optional_params = {} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Supported parameters are') as exc_info: self.config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, @@ -171,7 +171,7 @@ class TestRecraftImageGenerationTransformation: mock_get_secret.return_value = None headers = {} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='RECRAFT_API_KEY is not set') as exc_info: self.config.validate_environment( headers=headers, model=self.model, @@ -248,7 +248,7 @@ class TestRecraftImageGenerationTransformation: model_response = ImageResponse(data=[]) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Error transforming image generation response: Invalid JSON') as exc_info: self.config.transform_image_generation_response( model=self.model, raw_response=mock_response, diff --git a/tests/test_litellm/llms/stability/image_generation/test_stability_image_generation.py b/tests/test_litellm/llms/stability/image_generation/test_stability_image_generation.py index 6f1a04e78d3..c5b3c8fbdc5 100644 --- a/tests/test_litellm/llms/stability/image_generation/test_stability_image_generation.py +++ b/tests/test_litellm/llms/stability/image_generation/test_stability_image_generation.py @@ -83,7 +83,7 @@ class TestStabilityImageGenerationConfig: non_default_params = {"unsupported_param": "value"} optional_params = {} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="Supported parameters are \\['n', 'size',") as exc_info: self.config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, @@ -168,7 +168,7 @@ class TestStabilityImageGenerationConfig: def test_validate_environment_raises_without_api_key(self): """Test that validate_environment raises error without API key""" - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='STABILITY_API_KEY is not set\\. Please set it via') as exc_info: self.config.validate_environment( headers={}, model="stability/sd3", @@ -251,7 +251,7 @@ class TestStabilityImageGenerationConfig: model_response = ImageResponse(data=[]) mock_logging = MagicMock() - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Content was filtered by Stability AI safety systems') as exc_info: self.config.transform_image_generation_response( model="stability/sd3", raw_response=mock_response, diff --git a/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py b/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py index 2dcccb8ea7e..69afbb416aa 100644 --- a/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py +++ b/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py @@ -697,7 +697,7 @@ class TestErrorHandling: } } mock_response = _make_mock_response(body, status_code=400) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='TinyFish Search: query is required\\. See https') as exc_info: config.transform_search_response( raw_response=mock_response, logging_obj=None ) @@ -713,7 +713,7 @@ class TestErrorHandling: mock_response = _make_mock_response( body, status_code=429, headers={"Retry-After": "60"} ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='TinyFish Search: rate limit exceeded\\. See https') as exc_info: config.transform_search_response( raw_response=mock_response, logging_obj=None ) @@ -728,7 +728,7 @@ class TestErrorHandling: config = TinyfishSearchConfig() body = {"errors": [{"code": "10000", "message": "Internal"}]} mock_response = _make_mock_response(body, status_code=502) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='TinyFish Search') as exc_info: config.transform_search_response( raw_response=mock_response, logging_obj=None ) @@ -742,7 +742,7 @@ class TestErrorHandling: mock_response = _make_mock_response( json_data=None, status_code=502, text="Bad Gateway" ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='TinyFish Search: Bad Gateway<') as exc_info: config.transform_search_response( raw_response=mock_response, logging_obj=None ) @@ -756,7 +756,7 @@ class TestErrorHandling: mock_response = _make_mock_response( json_data=None, status_code=200, text="not json" ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='TinyFish Search: Expected JSON response, got: not json\\.') as exc_info: config.transform_search_response( raw_response=mock_response, logging_obj=None ) @@ -785,7 +785,7 @@ class TestErrorHandling: # check TinyFish's schema, not their own input. config = TinyfishSearchConfig() mock_response = _make_mock_response({"query": "x"}) # no `results` key - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='validation error for SearchResponse') as exc_info: config.transform_search_response( raw_response=mock_response, logging_obj=None ) diff --git a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_integration.py b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_integration.py index 272565990bd..8f9acafa49d 100644 --- a/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_integration.py +++ b/tests/test_litellm/llms/vertex_ai/files/test_vertex_ai_files_integration.py @@ -159,7 +159,7 @@ class TestVertexAIFilesIntegration: # This test ensures the type annotations and error messages include vertex_ai # Test that calling with unsupported provider raises appropriate error - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="unsupported_provider' is not a valid LlmProviders") as exc_info: litellm.file_content( file_id="test-file-id", custom_llm_provider="unsupported_provider", # This should fail diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index b83d4742b64..c189cdd0ea7 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -33,7 +33,7 @@ def test_validate_vertex_location_accepts_valid(location): ["attacker.example/", "evil.com#", "us.attacker.example", "us/../..", "US", "us_central1", "-us", "", None], ) def test_validate_vertex_location_rejects_invalid(location): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="vertex_location is required|Invalid vertex_location format"): validate_vertex_location(location) diff --git a/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py b/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py index 891d1c15c61..7922331d19f 100644 --- a/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py +++ b/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py @@ -137,7 +137,7 @@ class TestVolcengineResponsesAPITransformation: monkeypatch.delenv("ARK_API_KEY", raising=False) monkeypatch.delenv("VOLCENGINE_API_KEY", raising=False) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Volcengine API key is required\\. Set ARK_API_KEY /'): config.validate_environment(headers={}, model="volcengine/demo", litellm_params={}) def test_unsupported_params_are_dropped_with_extra_body(self): diff --git a/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py b/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py index 07298f03f86..1670dac0e9d 100644 --- a/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py +++ b/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py @@ -202,7 +202,7 @@ def test_volcengine_embedding_error_scenarios(): k: v for k, v in scenario.items() if k != "expected_error_pattern" } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match=f"(?i){scenario['expected_error_pattern']}") as exc_info: litellm.embedding(input=["test"], **test_params) # Verify error message contains expected pattern diff --git a/tests/test_litellm/llms/voyage/rerank/test_voyage_rerank_transformation.py b/tests/test_litellm/llms/voyage/rerank/test_voyage_rerank_transformation.py index 8f99609e3f5..f466b7e19b5 100644 --- a/tests/test_litellm/llms/voyage/rerank/test_voyage_rerank_transformation.py +++ b/tests/test_litellm/llms/voyage/rerank/test_voyage_rerank_transformation.py @@ -227,7 +227,7 @@ class TestVoyageRerankTransform: mock_logging = MagicMock() model_response = RerankResponse() - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Unauthorized') as exc_info: self.config.transform_rerank_response( model=self.model, raw_response=mock_response, @@ -248,7 +248,7 @@ class TestVoyageRerankTransform: mock_logging = MagicMock() model_response = RerankResponse() - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Failed to parse response: Invalid JSON response') as exc_info: self.config.transform_rerank_response( model=self.model, raw_response=mock_response, diff --git a/tests/test_litellm/llms/voyage/test_voyage_multimodal_embedding.py b/tests/test_litellm/llms/voyage/test_voyage_multimodal_embedding.py index f283e7fe0df..f3e6885cbe6 100644 --- a/tests/test_litellm/llms/voyage/test_voyage_multimodal_embedding.py +++ b/tests/test_litellm/llms/voyage/test_voyage_multimodal_embedding.py @@ -195,7 +195,7 @@ class TestVoyageMultimodalEmbeddings: monkeypatch.setattr(module, "get_secret_str", lambda name: None) config = VoyageMultimodalEmbeddingConfig() - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Voyage API key is required for multimodal embeddings\\. Set') as exc_info: config.validate_environment( {}, "voyage-multimodal-3.5", [], {}, {}, api_key=None ) @@ -207,7 +207,7 @@ class TestVoyageMultimodalEmbeddings: ) config = VoyageMultimodalEmbeddingConfig() - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Voyage multimodal embeddings require a non-empty') as exc_info: config._normalize_content_item({"type": "image_url", "image_url": {}}) assert "image_url" in str(exc_info.value) diff --git a/tests/test_litellm/llms/xai/test_xai_key_fallback.py b/tests/test_litellm/llms/xai/test_xai_key_fallback.py index 4c769c572ac..ec3eb83309c 100644 --- a/tests/test_litellm/llms/xai/test_xai_key_fallback.py +++ b/tests/test_litellm/llms/xai/test_xai_key_fallback.py @@ -168,7 +168,7 @@ def test_responses_config_raises_when_no_key_is_available(monkeypatch): monkeypatch.setattr(litellm, "api_key", None) monkeypatch.delenv("XAI_API_KEY", raising=False) - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='XAI API key is required\\. Set api_key, litellm\\.xai_key') as exc_info: XAIResponsesAPIConfig().validate_environment({}, "xai/grok-3-mini", None) error_message = str(exc_info.value) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 0209abee510..d5936b2ae86 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -8185,7 +8185,7 @@ class TestGetUserObjectPermission: return_value=None, ), ): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="user 'human-dangling' names object_permission_id"): await MCPRequestHandler._get_user_object_permission(auth) async def test_no_user_id_places_no_ceiling(self): diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 34852850de6..b4d3782ba43 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -2695,13 +2695,6 @@ async def test_token_endpoint_respects_x_forwarded_host(): "443", "https://internal.local", ), - ( - "http://localhost:4000/", - "https", - "proxy.example.com", - "8443", - "https://proxy.example.com:8443", - ), ( "http://localhost:4000/", "https", diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_short_mcp_tool_prefix.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_short_mcp_tool_prefix.py index 6e3ac014840..941e5deee93 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_short_mcp_tool_prefix.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_short_mcp_tool_prefix.py @@ -79,7 +79,7 @@ class TestShortPrefixHelpers: assert compute_short_server_prefix("abc") != compute_short_server_prefix("abd") def test_short_prefix_requires_server_id(self): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='compute_short_server_prefix requires a non-empty server_id'): compute_short_server_prefix("") def test_flag_defaults_to_false(self): diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 6b40fa1b324..762d2cbf3c7 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -268,7 +268,7 @@ def test_get_experimental_ui_login_jwt_auth_token_invalid( invalid_sso_user_defined_values, ): """Test generating JWT token with missing user role""" - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='User role is required for experimental UI login') as exc_info: ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token( invalid_sso_user_defined_values ) @@ -883,7 +883,7 @@ async def test_get_user_object_wraps_db_outage_as_valueerror_preserving_context( mock_cache.async_set_cache = AsyncMock() with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match="User doesn't exist in db\\.") as exc_info: await get_user_object( user_id="outage-contract-probe-user", prisma_client=mock_prisma_client, diff --git a/tests/test_litellm/proxy/auth/test_auth_hot_path_network_requests.py b/tests/test_litellm/proxy/auth/test_auth_hot_path_network_requests.py index 22752f767ce..6b2d2babedc 100644 --- a/tests/test_litellm/proxy/auth/test_auth_hot_path_network_requests.py +++ b/tests/test_litellm/proxy/auth/test_auth_hot_path_network_requests.py @@ -1,521 +1,521 @@ -""" -Test to count and track the number of network requests (DB queries, cache lookups) -made on the hot path for keys that have team_id and user_id attached. - -This test ensures we don't regress on the number of network requests made during -request authentication, which directly impacts proxy latency. - -The hot path covers auth functions called on every LLM API request: -- get_key_object: lookup the API key -- get_team_object: lookup the team (for keys with team_id) -- get_user_object: lookup the user (for keys with user_id) -- get_team_membership: lookup team member budget (when team_member_spend set) - -Each function does: cache read -> (on miss) DB query -> cache write. -We count these to catch regressions in the number of network requests. - -NOTE: This test does NOT require proxy extras (apscheduler, etc.) because -it tests at the auth_checks level, not the full proxy_server level. -""" - -import os -import sys -import time -from typing import Any, Dict, List, Optional -from unittest.mock import AsyncMock, MagicMock - -import pytest - -sys.path.insert(0, os.path.abspath("../../..")) - -from litellm.caching.dual_cache import DualCache -from litellm.caching.in_memory_cache import InMemoryCache -from litellm.proxy._types import ( - LiteLLM_TeamTableCachedObj, - LiteLLM_UserTable, - LitellmUserRoles, - UserAPIKeyAuth, - LiteLLM_TeamMembership, - hash_token, -) -from litellm.proxy.auth.auth_checks import ( - get_key_object, - get_team_membership, - get_team_object, - get_user_object, -) - - -class CacheCallTracker: - """ - Tracks cache read/write operations by wrapping DualCache methods. - This is used to count network-level operations on the hot path. - """ - - def __init__(self): - self.cache_reads: List[Dict[str, Any]] = [] - self.cache_writes: List[Dict[str, Any]] = [] - self.db_queries: List[Dict[str, Any]] = [] - - def get_summary(self) -> Dict[str, Any]: - return { - "total_cache_reads": len(self.cache_reads), - "total_cache_writes": len(self.cache_writes), - "total_db_queries": len(self.db_queries), - "total_network_requests": len(self.cache_reads) - + len(self.cache_writes) - + len(self.db_queries), - "cache_read_keys": [r["key"] for r in self.cache_reads], - "cache_write_keys": [w["key"] for w in self.cache_writes], - "db_query_details": self.db_queries, - } - - -def _wrap_cache_with_tracker(cache: DualCache, tracker: CacheCallTracker) -> DualCache: - """Wrap a DualCache to track all reads and writes.""" - original_async_get = cache.async_get_cache - original_async_set = cache.async_set_cache - - async def tracked_async_get(key, *args, **kwargs): - result = await original_async_get(key, *args, **kwargs) - tracker.cache_reads.append( - {"key": key, "hit": result is not None, "method": "async_get_cache"} - ) - return result - - async def tracked_async_set(key, value, *args, **kwargs): - tracker.cache_writes.append({"key": key, "method": "async_set_cache"}) - return await original_async_set(key, value, *args, **kwargs) - - cache.async_get_cache = tracked_async_get - cache.async_set_cache = tracked_async_set - return cache - - -def _create_valid_token( - api_key: str, - team_id: str, - user_id: str, - has_team_member_spend: bool = False, - org_id: Optional[str] = None, -) -> UserAPIKeyAuth: - """Create a UserAPIKeyAuth with team_id and user_id set.""" - hashed = hash_token(api_key) - return UserAPIKeyAuth( - token=hashed, - api_key=api_key, - team_id=team_id, - user_id=user_id, - org_id=org_id, - models=["gpt-4", "gpt-3.5-turbo"], - max_budget=100.0, - spend=10.0, - team_spend=50.0, - team_max_budget=1000.0, - team_models=["gpt-4", "gpt-3.5-turbo"], - team_member_spend=5.0 if has_team_member_spend else None, - last_refreshed_at=time.time(), - user_role=LitellmUserRoles.INTERNAL_USER, - ) - - -def _create_team_object(team_id: str) -> LiteLLM_TeamTableCachedObj: - """Create a team table object for caching.""" - return LiteLLM_TeamTableCachedObj( - team_id=team_id, - models=["gpt-4", "gpt-3.5-turbo"], - max_budget=1000.0, - spend=50.0, - tpm_limit=10000, - rpm_limit=100, - last_refreshed_at=time.time(), - ) - - -def _create_user_object(user_id: str) -> LiteLLM_UserTable: - """Create a user table object for caching.""" - return LiteLLM_UserTable( - user_id=user_id, - max_budget=500.0, - spend=25.0, - models=["gpt-4"], - tpm_limit=5000, - rpm_limit=50, - user_role=LitellmUserRoles.INTERNAL_USER, - user_email="test@example.com", - ) - - -# ============================================================================ -# TEST: get_key_object cache behavior -# ============================================================================ - - -@pytest.mark.asyncio -async def test_get_key_object_warm_cache(): - """ - Test get_key_object with a warm cache - should hit cache, no DB query. - """ - api_key = "sk-test-key-warm" - team_id = "team-123" - user_id = "user-456" - hashed_token = hash_token(api_key) - - valid_token = _create_valid_token(api_key, team_id, user_id) - - # Create cache with pre-populated data - cache = DualCache(in_memory_cache=InMemoryCache()) - await cache.async_set_cache(key=hashed_token, value=valid_token) - - # Track cache operations - tracker = CacheCallTracker() - tracked_cache = _wrap_cache_with_tracker(cache, tracker) - - # Mock prisma client (should NOT be called for warm cache) - mock_prisma = MagicMock() - mock_prisma.get_data = AsyncMock() - - result = await get_key_object( - hashed_token=hashed_token, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - ) - - summary = tracker.get_summary() - - # Should have exactly 1 cache read - assert summary["total_cache_reads"] == 1 - assert hashed_token in summary["cache_read_keys"] - - # Prisma should NOT have been called - mock_prisma.get_data.assert_not_called() - - # Result should be the cached token - assert result.token == hashed_token - - -@pytest.mark.asyncio -async def test_get_key_object_cold_cache(): - """ - Test get_key_object with a cold cache - should miss cache, query DB. - """ - api_key = "sk-test-key-cold" - team_id = "team-123" - user_id = "user-456" - hashed_token = hash_token(api_key) - - valid_token = _create_valid_token(api_key, team_id, user_id) - - # Create empty cache - cache = DualCache(in_memory_cache=InMemoryCache()) - - tracker = CacheCallTracker() - tracked_cache = _wrap_cache_with_tracker(cache, tracker) - - # Mock prisma client to return token on DB query - mock_prisma = MagicMock() - mock_prisma.get_data = AsyncMock(return_value=valid_token) - - await get_key_object( - hashed_token=hashed_token, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - ) - - summary = tracker.get_summary() - - # Should have 1 cache read (miss) and at least 1 cache write (populate cache) - assert summary["total_cache_reads"] >= 1 - - # Prisma SHOULD have been called - mock_prisma.get_data.assert_called_once() - - -# ============================================================================ -# TEST: get_team_object cache behavior -# ============================================================================ - - -@pytest.mark.asyncio -async def test_get_team_object_warm_cache(): - """ - Test get_team_object with a warm cache - should hit cache, no DB query. - """ - team_id = "team-warm-123" - team_obj = _create_team_object(team_id) - - cache = DualCache(in_memory_cache=InMemoryCache()) - cache_key = f"team_id:{team_id}" - await cache.async_set_cache(key=cache_key, value=team_obj) - - tracker = CacheCallTracker() - tracked_cache = _wrap_cache_with_tracker(cache, tracker) - - mock_prisma = MagicMock() - mock_prisma.db = MagicMock() - mock_prisma.db.litellm_teamtable = MagicMock() - mock_prisma.db.litellm_teamtable.find_unique = AsyncMock() - - await get_team_object( - team_id=team_id, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - ) - - summary = tracker.get_summary() - - assert summary["total_cache_reads"] >= 1 - assert cache_key in summary["cache_read_keys"] - - # DB should NOT have been called - mock_prisma.db.litellm_teamtable.find_unique.assert_not_called() - - -# ============================================================================ -# TEST: get_user_object cache behavior -# ============================================================================ - - -@pytest.mark.asyncio -async def test_get_user_object_warm_cache(): - """ - Test get_user_object with a warm cache - should hit cache, no DB query. - """ - user_id = "user-warm-456" - user_obj = _create_user_object(user_id) - - cache = DualCache(in_memory_cache=InMemoryCache()) - await cache.async_set_cache(key=user_id, value=user_obj) - - tracker = CacheCallTracker() - tracked_cache = _wrap_cache_with_tracker(cache, tracker) - - mock_prisma = MagicMock() - mock_prisma.db = MagicMock() - mock_prisma.db.litellm_usertable = MagicMock() - mock_prisma.db.litellm_usertable.find_unique = AsyncMock() - - await get_user_object( - user_id=user_id, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - user_id_upsert=False, - ) - - summary = tracker.get_summary() - - assert summary["total_cache_reads"] >= 1 - assert user_id in summary["cache_read_keys"] - - # DB should NOT have been called - mock_prisma.db.litellm_usertable.find_unique.assert_not_called() - - -# ============================================================================ -# TEST: get_team_membership cache behavior -# ============================================================================ - - -@pytest.mark.asyncio -async def test_get_team_membership_warm_cache(): - """ - Test get_team_membership with a warm cache - should hit cache, no DB query. - """ - user_id = "user-tm-456" - team_id = "team-tm-123" - - membership_dict = { - "user_id": user_id, - "team_id": team_id, - "spend": 3.0, - "budget_id": None, - "litellm_budget_table": None, - } - - cache = DualCache(in_memory_cache=InMemoryCache()) - # Cache key format used by get_team_membership - cache_key = f"team_membership:{user_id}:{team_id}" - await cache.async_set_cache(key=cache_key, value=membership_dict) - - tracker = CacheCallTracker() - tracked_cache = _wrap_cache_with_tracker(cache, tracker) - - mock_prisma = MagicMock() - mock_prisma.db = MagicMock() - mock_prisma.db.litellm_teammembership = MagicMock() - mock_prisma.db.litellm_teammembership.find_unique = AsyncMock() - - await get_team_membership( - user_id=user_id, - team_id=team_id, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - ) - - summary = tracker.get_summary() - - assert summary["total_cache_reads"] >= 1 - assert cache_key in summary["cache_read_keys"] - - # DB should NOT have been called - mock_prisma.db.litellm_teammembership.find_unique.assert_not_called() - - -# ============================================================================ -# TEST: Document duplicate team membership cache key issue -# ============================================================================ - - -@pytest.mark.asyncio -async def test_team_membership_cache_key_duplication(): - """ - Document the team membership duplicate cache key issue: - - Team membership is queried via TWO different cache keys: - 1. "{team_id}_{user_id}" - used in user_api_key_auth.py:1048 - 2. "team_membership:{user_id}:{team_id}" - used in auth_checks.py:960 (get_team_membership) - - This test documents that both keys refer to the same data but use different - cache key formats, potentially leading to duplicate lookups. - """ - user_id = "user-dup-456" - team_id = "team-dup-123" - - # The two different cache keys used for the same data - key_format_1 = f"{team_id}_{user_id}" # user_api_key_auth format - key_format_2 = f"team_membership:{user_id}:{team_id}" # auth_checks format - - _ = { - "user_id": user_id, - "team_id": team_id, - "spend": 3.0, - } - - # Document that these are different keys - assert ( - key_format_1 != key_format_2 - ), "Cache keys should be different (this is the bug)" - - # Document that these are different keys - assert ( - key_format_1 != key_format_2 - ), "Cache keys should be different (this is the bug)" - - -# ============================================================================ -# TEST: Full hot path network count summary -# ============================================================================ - - -@pytest.mark.asyncio -async def test_full_hot_path_network_count(): - """ - Summary test that counts all network operations when processing - a request with a key that has team_id and user_id attached. - - This test verifies the baseline number of cache operations expected - on a fully warm cache path. - """ - api_key = "sk-test-full-path" - team_id = "team-full-123" - user_id = "user-full-456" - hashed_token = hash_token(api_key) - - # Create all objects - valid_token = _create_valid_token( - api_key, team_id, user_id, has_team_member_spend=True - ) - team_obj = _create_team_object(team_id) - user_obj = _create_user_object(user_id) - membership_data = LiteLLM_TeamMembership( - user_id=user_id, - team_id=team_id, - spend=3.0, - budget_id=None, - litellm_budget_table=None, - ) - - # Pre-populate cache with all data - cache = DualCache(in_memory_cache=InMemoryCache()) - await cache.async_set_cache(key=hashed_token, value=valid_token) - await cache.async_set_cache(key=f"team_id:{team_id}", value=team_obj) - await cache.async_set_cache(key=user_id, value=user_obj) - await cache.async_set_cache( - key=f"team_membership:{user_id}:{team_id}", value=membership_data.model_dump() - ) - await cache.async_set_cache( - key=f"{team_id}_{user_id}", value=membership_data.model_dump() - ) - - # Create tracker AFTER populating cache - tracker = CacheCallTracker() - tracked_cache = _wrap_cache_with_tracker(cache, tracker) - - # Mock prisma (should not be called on warm cache) - mock_prisma = MagicMock() - - # Call each function to simulate the hot path - await get_key_object( - hashed_token=hashed_token, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - ) - - await get_team_object( - team_id=team_id, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - ) - - await get_user_object( - user_id=user_id, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - user_id_upsert=False, - ) - - await get_team_membership( - user_id=user_id, - team_id=team_id, - prisma_client=mock_prisma, - user_api_key_cache=tracked_cache, - parent_otel_span=None, - proxy_logging_obj=None, - ) - - summary = tracker.get_summary() - - # Assertions for expected baseline - # On warm cache: 4 reads (key, team, user, team_membership) - assert ( - summary["total_cache_reads"] == 4 - ), f"Expected 4 cache reads on warm path, got {summary['total_cache_reads']}" - - # No DB queries on warm cache - assert ( - summary["total_db_queries"] == 0 - ), f"Expected 0 DB queries on warm path, got {summary['total_db_queries']}" - - # Total network requests should be exactly 4 on warm cache - assert ( - summary["total_network_requests"] == 4 - ), f"Expected 4 total network requests on warm path, got {summary['total_network_requests']}" +""" +Test to count and track the number of network requests (DB queries, cache lookups) +made on the hot path for keys that have team_id and user_id attached. + +This test ensures we don't regress on the number of network requests made during +request authentication, which directly impacts proxy latency. + +The hot path covers auth functions called on every LLM API request: +- get_key_object: lookup the API key +- get_team_object: lookup the team (for keys with team_id) +- get_user_object: lookup the user (for keys with user_id) +- get_team_membership: lookup team member budget (when team_member_spend set) + +Each function does: cache read -> (on miss) DB query -> cache write. +We count these to catch regressions in the number of network requests. + +NOTE: This test does NOT require proxy extras (apscheduler, etc.) because +it tests at the auth_checks level, not the full proxy_server level. +""" + +import os +import sys +import time +from typing import Any, Dict, List, Optional +from unittest.mock import AsyncMock, MagicMock + +import pytest + +sys.path.insert(0, os.path.abspath("../../..")) + +from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.proxy._types import ( + LiteLLM_TeamTableCachedObj, + LiteLLM_UserTable, + LitellmUserRoles, + UserAPIKeyAuth, + LiteLLM_TeamMembership, + hash_token, +) +from litellm.proxy.auth.auth_checks import ( + get_key_object, + get_team_membership, + get_team_object, + get_user_object, +) + + +class CacheCallTracker: + """ + Tracks cache read/write operations by wrapping DualCache methods. + This is used to count network-level operations on the hot path. + """ + + def __init__(self): + self.cache_reads: List[Dict[str, Any]] = [] + self.cache_writes: List[Dict[str, Any]] = [] + self.db_queries: List[Dict[str, Any]] = [] + + def get_summary(self) -> Dict[str, Any]: + return { + "total_cache_reads": len(self.cache_reads), + "total_cache_writes": len(self.cache_writes), + "total_db_queries": len(self.db_queries), + "total_network_requests": len(self.cache_reads) + + len(self.cache_writes) + + len(self.db_queries), + "cache_read_keys": [r["key"] for r in self.cache_reads], + "cache_write_keys": [w["key"] for w in self.cache_writes], + "db_query_details": self.db_queries, + } + + +def _wrap_cache_with_tracker(cache: DualCache, tracker: CacheCallTracker) -> DualCache: + """Wrap a DualCache to track all reads and writes.""" + original_async_get = cache.async_get_cache + original_async_set = cache.async_set_cache + + async def tracked_async_get(key, *args, **kwargs): + result = await original_async_get(key, *args, **kwargs) + tracker.cache_reads.append( + {"key": key, "hit": result is not None, "method": "async_get_cache"} + ) + return result + + async def tracked_async_set(key, value, *args, **kwargs): + tracker.cache_writes.append({"key": key, "method": "async_set_cache"}) + return await original_async_set(key, value, *args, **kwargs) + + cache.async_get_cache = tracked_async_get + cache.async_set_cache = tracked_async_set + return cache + + +def _create_valid_token( + api_key: str, + team_id: str, + user_id: str, + has_team_member_spend: bool = False, + org_id: Optional[str] = None, +) -> UserAPIKeyAuth: + """Create a UserAPIKeyAuth with team_id and user_id set.""" + hashed = hash_token(api_key) + return UserAPIKeyAuth( + token=hashed, + api_key=api_key, + team_id=team_id, + user_id=user_id, + org_id=org_id, + models=["gpt-4", "gpt-3.5-turbo"], + max_budget=100.0, + spend=10.0, + team_spend=50.0, + team_max_budget=1000.0, + team_models=["gpt-4", "gpt-3.5-turbo"], + team_member_spend=5.0 if has_team_member_spend else None, + last_refreshed_at=time.time(), + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + +def _create_team_object(team_id: str) -> LiteLLM_TeamTableCachedObj: + """Create a team table object for caching.""" + return LiteLLM_TeamTableCachedObj( + team_id=team_id, + models=["gpt-4", "gpt-3.5-turbo"], + max_budget=1000.0, + spend=50.0, + tpm_limit=10000, + rpm_limit=100, + last_refreshed_at=time.time(), + ) + + +def _create_user_object(user_id: str) -> LiteLLM_UserTable: + """Create a user table object for caching.""" + return LiteLLM_UserTable( + user_id=user_id, + max_budget=500.0, + spend=25.0, + models=["gpt-4"], + tpm_limit=5000, + rpm_limit=50, + user_role=LitellmUserRoles.INTERNAL_USER, + user_email="test@example.com", + ) + + +# ============================================================================ +# TEST: get_key_object cache behavior +# ============================================================================ + + +@pytest.mark.asyncio +async def test_get_key_object_warm_cache(): + """ + Test get_key_object with a warm cache - should hit cache, no DB query. + """ + api_key = "sk-test-key-warm" + team_id = "team-123" + user_id = "user-456" + hashed_token = hash_token(api_key) + + valid_token = _create_valid_token(api_key, team_id, user_id) + + # Create cache with pre-populated data + cache = DualCache(in_memory_cache=InMemoryCache()) + await cache.async_set_cache(key=hashed_token, value=valid_token) + + # Track cache operations + tracker = CacheCallTracker() + tracked_cache = _wrap_cache_with_tracker(cache, tracker) + + # Mock prisma client (should NOT be called for warm cache) + mock_prisma = MagicMock() + mock_prisma.get_data = AsyncMock() + + result = await get_key_object( + hashed_token=hashed_token, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + ) + + summary = tracker.get_summary() + + # Should have exactly 1 cache read + assert summary["total_cache_reads"] == 1 + assert hashed_token in summary["cache_read_keys"] + + # Prisma should NOT have been called + mock_prisma.get_data.assert_not_called() + + # Result should be the cached token + assert result.token == hashed_token + + +@pytest.mark.asyncio +async def test_get_key_object_cold_cache(): + """ + Test get_key_object with a cold cache - should miss cache, query DB. + """ + api_key = "sk-test-key-cold" + team_id = "team-123" + user_id = "user-456" + hashed_token = hash_token(api_key) + + valid_token = _create_valid_token(api_key, team_id, user_id) + + # Create empty cache + cache = DualCache(in_memory_cache=InMemoryCache()) + + tracker = CacheCallTracker() + tracked_cache = _wrap_cache_with_tracker(cache, tracker) + + # Mock prisma client to return token on DB query + mock_prisma = MagicMock() + mock_prisma.get_data = AsyncMock(return_value=valid_token) + + await get_key_object( + hashed_token=hashed_token, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + ) + + summary = tracker.get_summary() + + # Should have 1 cache read (miss) and at least 1 cache write (populate cache) + assert summary["total_cache_reads"] >= 1 + + # Prisma SHOULD have been called + mock_prisma.get_data.assert_called_once() + + +# ============================================================================ +# TEST: get_team_object cache behavior +# ============================================================================ + + +@pytest.mark.asyncio +async def test_get_team_object_warm_cache(): + """ + Test get_team_object with a warm cache - should hit cache, no DB query. + """ + team_id = "team-warm-123" + team_obj = _create_team_object(team_id) + + cache = DualCache(in_memory_cache=InMemoryCache()) + cache_key = f"team_id:{team_id}" + await cache.async_set_cache(key=cache_key, value=team_obj) + + tracker = CacheCallTracker() + tracked_cache = _wrap_cache_with_tracker(cache, tracker) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_teamtable = MagicMock() + mock_prisma.db.litellm_teamtable.find_unique = AsyncMock() + + await get_team_object( + team_id=team_id, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + ) + + summary = tracker.get_summary() + + assert summary["total_cache_reads"] >= 1 + assert cache_key in summary["cache_read_keys"] + + # DB should NOT have been called + mock_prisma.db.litellm_teamtable.find_unique.assert_not_called() + + +# ============================================================================ +# TEST: get_user_object cache behavior +# ============================================================================ + + +@pytest.mark.asyncio +async def test_get_user_object_warm_cache(): + """ + Test get_user_object with a warm cache - should hit cache, no DB query. + """ + user_id = "user-warm-456" + user_obj = _create_user_object(user_id) + + cache = DualCache(in_memory_cache=InMemoryCache()) + await cache.async_set_cache(key=user_id, value=user_obj) + + tracker = CacheCallTracker() + tracked_cache = _wrap_cache_with_tracker(cache, tracker) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_usertable = MagicMock() + mock_prisma.db.litellm_usertable.find_unique = AsyncMock() + + await get_user_object( + user_id=user_id, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + user_id_upsert=False, + ) + + summary = tracker.get_summary() + + assert summary["total_cache_reads"] >= 1 + assert user_id in summary["cache_read_keys"] + + # DB should NOT have been called + mock_prisma.db.litellm_usertable.find_unique.assert_not_called() + + +# ============================================================================ +# TEST: get_team_membership cache behavior +# ============================================================================ + + +@pytest.mark.asyncio +async def test_get_team_membership_warm_cache(): + """ + Test get_team_membership with a warm cache - should hit cache, no DB query. + """ + user_id = "user-tm-456" + team_id = "team-tm-123" + + membership_dict = { + "user_id": user_id, + "team_id": team_id, + "spend": 3.0, + "budget_id": None, + "litellm_budget_table": None, + } + + cache = DualCache(in_memory_cache=InMemoryCache()) + # Cache key format used by get_team_membership + cache_key = f"team_membership:{user_id}:{team_id}" + await cache.async_set_cache(key=cache_key, value=membership_dict) + + tracker = CacheCallTracker() + tracked_cache = _wrap_cache_with_tracker(cache, tracker) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_teammembership = MagicMock() + mock_prisma.db.litellm_teammembership.find_unique = AsyncMock() + + await get_team_membership( + user_id=user_id, + team_id=team_id, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + ) + + summary = tracker.get_summary() + + assert summary["total_cache_reads"] >= 1 + assert cache_key in summary["cache_read_keys"] + + # DB should NOT have been called + mock_prisma.db.litellm_teammembership.find_unique.assert_not_called() + + +# ============================================================================ +# TEST: Document duplicate team membership cache key issue +# ============================================================================ + + +@pytest.mark.asyncio +async def test_team_membership_cache_key_duplication(): + """ + Document the team membership duplicate cache key issue: + + Team membership is queried via TWO different cache keys: + 1. "{team_id}_{user_id}" - used in user_api_key_auth.py:1048 + 2. "team_membership:{user_id}:{team_id}" - used in auth_checks.py:960 (get_team_membership) + + This test documents that both keys refer to the same data but use different + cache key formats, potentially leading to duplicate lookups. + """ + user_id = "user-dup-456" + team_id = "team-dup-123" + + # The two different cache keys used for the same data + key_format_1 = f"{team_id}_{user_id}" # user_api_key_auth format + key_format_2 = f"team_membership:{user_id}:{team_id}" # auth_checks format + + _ = { + "user_id": user_id, + "team_id": team_id, + "spend": 3.0, + } + + # Document that these are different keys + assert ( + key_format_1 != key_format_2 + ), "Cache keys should be different (this is the bug)" + + # Document that these are different keys + assert ( + key_format_1 != key_format_2 + ), "Cache keys should be different (this is the bug)" + + +# ============================================================================ +# TEST: Full hot path network count summary +# ============================================================================ + + +@pytest.mark.asyncio +async def test_full_hot_path_network_count(): + """ + Summary test that counts all network operations when processing + a request with a key that has team_id and user_id attached. + + This test verifies the baseline number of cache operations expected + on a fully warm cache path. + """ + api_key = "sk-test-full-path" + team_id = "team-full-123" + user_id = "user-full-456" + hashed_token = hash_token(api_key) + + # Create all objects + valid_token = _create_valid_token( + api_key, team_id, user_id, has_team_member_spend=True + ) + team_obj = _create_team_object(team_id) + user_obj = _create_user_object(user_id) + membership_data = LiteLLM_TeamMembership( + user_id=user_id, + team_id=team_id, + spend=3.0, + budget_id=None, + litellm_budget_table=None, + ) + + # Pre-populate cache with all data + cache = DualCache(in_memory_cache=InMemoryCache()) + await cache.async_set_cache(key=hashed_token, value=valid_token) + await cache.async_set_cache(key=f"team_id:{team_id}", value=team_obj) + await cache.async_set_cache(key=user_id, value=user_obj) + await cache.async_set_cache( + key=f"team_membership:{user_id}:{team_id}", value=membership_data.model_dump() + ) + await cache.async_set_cache( + key=f"{team_id}_{user_id}", value=membership_data.model_dump() + ) + + # Create tracker AFTER populating cache + tracker = CacheCallTracker() + tracked_cache = _wrap_cache_with_tracker(cache, tracker) + + # Mock prisma (should not be called on warm cache) + mock_prisma = MagicMock() + + # Call each function to simulate the hot path + await get_key_object( + hashed_token=hashed_token, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + ) + + await get_team_object( + team_id=team_id, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + ) + + await get_user_object( + user_id=user_id, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + user_id_upsert=False, + ) + + await get_team_membership( + user_id=user_id, + team_id=team_id, + prisma_client=mock_prisma, + user_api_key_cache=tracked_cache, + parent_otel_span=None, + proxy_logging_obj=None, + ) + + summary = tracker.get_summary() + + # Assertions for expected baseline + # On warm cache: 4 reads (key, team, user, team_membership) + assert ( + summary["total_cache_reads"] == 4 + ), f"Expected 4 cache reads on warm path, got {summary['total_cache_reads']}" + + # No DB queries on warm cache + assert ( + summary["total_db_queries"] == 0 + ), f"Expected 0 DB queries on warm path, got {summary['total_db_queries']}" + + # Total network requests should be exactly 4 on warm cache + assert ( + summary["total_network_requests"] == 4 + ), f"Expected 4 total network requests on warm path, got {summary['total_network_requests']}" # ============================================================================ @@ -540,7 +540,7 @@ async def test_get_user_object_missing_user_negative_cache(): mock_prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=None) for _ in range(3): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="User doesn't exist in db\\."): await get_user_object( user_id=user_id, prisma_client=mock_prisma, @@ -570,7 +570,7 @@ async def test_get_user_object_missing_user_rechecks_after_expiry(): mock_prisma.db.litellm_usertable = MagicMock() mock_prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=None) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="User doesn't exist in db\\."): await get_user_object( user_id=user_id, prisma_client=mock_prisma, @@ -586,7 +586,7 @@ async def test_get_user_object_missing_user_rechecks_after_expiry(): time.time() - (db_cache_expiry + 1), ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="User doesn't exist in db\\."): await get_user_object( user_id=user_id, prisma_client=mock_prisma, diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index ecf7f89d487..9301176f3ed 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -1588,7 +1588,7 @@ class TestCheckCompleteCredentialsBlocksSSRF: "litellm.proxy.auth.auth_utils.validate_url", side_effect=SSRFError(f"blocked: {blocked_url}"), ): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='is rejected by the SSRF guard') as exc_info: check_complete_credentials( { "model": "gpt-4", @@ -2144,7 +2144,7 @@ class TestIsRequestBodySafeBlocksEndpointTargetingFields: ], ) def test_endpoint_targeting_field_in_request_body_is_rejected(self, field): - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request') as exc: is_request_body_safe( request_body={"model": "gpt-4", field: "https://attacker.example"}, general_settings={}, @@ -2165,7 +2165,7 @@ class TestIsRequestBodySafeBlocksEndpointTargetingFields: # on the blocklist into an SSRF / credential-exfil hole. Verify # that supplying an api_key (alongside the banned param) does NOT # bypass the gate — it can only be opened by an admin opt-in. - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request') as exc: is_request_body_safe( request_body={ "model": "gpt-4", @@ -2722,7 +2722,7 @@ class TestObservabilityCallbackBans: ], ) def test_observability_field_in_request_body_root_is_rejected(self, field): - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request') as exc: is_request_body_safe( request_body={"model": "gpt-4", field: "attacker-value"}, general_settings={}, @@ -2752,7 +2752,7 @@ class TestObservabilityCallbackBans: # Verifies the metadata walk: a value smuggled inside ``metadata`` # or ``litellm_metadata`` is just as dangerous as the same field # at the body root, and must hit the same gate. - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request') as exc: is_request_body_safe( request_body={ "model": "gpt-4", @@ -2787,7 +2787,7 @@ class TestObservabilityCallbackBans: ) def test_observability_field_in_litellm_params_metadata_is_rejected(self): - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request: turn_off_message_logging is not allowed') as exc: is_request_body_safe( request_body={ "model": "gpt-4", @@ -2814,7 +2814,7 @@ class TestObservabilityCallbackBans: # the ``isinstance(dict)`` guard. import json - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request: langfuse_host is not allowed in request') as exc: is_request_body_safe( request_body={ "model": "gpt-4", @@ -2887,7 +2887,7 @@ def test_model_level_allow_does_not_skip_subsequent_banned_params(monkeypatch): lambda model, param, request_body_value, llm_router: param == "api_base", ) - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request: langfuse_host is not allowed in request') as exc: is_request_body_safe( request_body={ "model": "gpt-4", @@ -2958,7 +2958,7 @@ class TestPricingInjectionBlocked: ], ) def test_pricing_field_rejected_by_default(self, field, value): - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='Rejected Request') as exc: is_request_body_safe( request_body={"model": "gpt-4", field: value}, general_settings={}, diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index a9e12beb54b..99a0a4c0a8b 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -2589,7 +2589,7 @@ async def test_find_and_validate_raises_when_required_team_not_found(): # Token without team info jwt_token = {"sub": "user-1"} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="No team found in token\\. Checked team_id field 'None' and") as exc_info: await JWTAuthManager.find_and_validate_specific_team_id( jwt_handler=jwt_handler, jwt_valid_token=jwt_token, @@ -2916,7 +2916,7 @@ async def test_find_and_validate_specific_team_id_hints_bracket_notation(): # token has roles as a list — dot-notation won't find anything token = {"roles": ["team1"]} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="is not supported\\. Use 'roles' instead — LiteLLM") as exc_info: await JWTAuthManager.find_and_validate_specific_team_id( jwt_handler=handler, jwt_valid_token=token, @@ -2947,7 +2947,7 @@ async def test_find_and_validate_specific_team_id_hints_bracket_index_notation() handler = _make_jwt_handler("roles[0]") token = {"roles": ["team1"]} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="is not supported in team_id_jwt_field\\. Use 'roles' instead") as exc_info: await JWTAuthManager.find_and_validate_specific_team_id( jwt_handler=handler, jwt_valid_token=token, @@ -2977,7 +2977,7 @@ async def test_find_and_validate_specific_team_id_no_hint_for_valid_field(): handler = _make_jwt_handler("appid") token = {} # no appid — triggers the "no team found" path - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="No team found in token\\. Checked team_id field 'appid' and") as exc_info: await JWTAuthManager.find_and_validate_specific_team_id( jwt_handler=handler, jwt_valid_token=token, @@ -4807,7 +4807,7 @@ async def test_multi_issuer_jwt_unknown_issuer_falls_back_to_global_jwks(monkeyp kid="issuer-key", ) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='Missing JWT Public Key URL from environment\\.') as exc: await jwt_handler.auth_jwt(token=token) assert "Missing JWT Public Key URL from environment." in str(exc.value) @@ -4838,7 +4838,7 @@ async def test_multi_issuer_jwt_rejects_wrong_audience(monkeypatch): kid="issuer-key", ) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match="Validation fails: Audience doesn't match") as exc: await jwt_handler.auth_jwt(token=token) assert "Validation fails" in str(exc.value) @@ -4881,7 +4881,7 @@ async def test_multi_issuer_jwt_same_kid_does_not_cross_issuer_keys(monkeypatch) kid=shared_kid, ) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='Validation fails: Signature verification failed') as exc: await jwt_handler.auth_jwt(token=token) assert "Validation fails" in str(exc.value) @@ -4936,7 +4936,7 @@ def test_multi_issuer_jwt_requires_audience_unless_explicitly_disabled( issuer = "https://issuer.example.com" jwks_url = f"{issuer}/keys" - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='must configure audience or set') as exc: LiteLLM_JWTAuth( issuers=[ { @@ -4953,7 +4953,7 @@ def test_multi_issuer_jwt_rejects_audience_with_disable_audience_validation(): issuer = "https://issuer.example.com" jwks_url = f"{issuer}/keys" - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='cannot set audience and disable_audience_validation=True') as exc: LiteLLM_JWTAuth( issuers=[ { diff --git a/tests/test_litellm/proxy/auth/test_oauth2_proxy_hook.py b/tests/test_litellm/proxy/auth/test_oauth2_proxy_hook.py index dcbfd281e01..2d81d48de1e 100644 --- a/tests/test_litellm/proxy/auth/test_oauth2_proxy_hook.py +++ b/tests/test_litellm/proxy/auth/test_oauth2_proxy_hook.py @@ -141,7 +141,7 @@ async def test_refuses_to_map_non_identity_fields(configure_proxy, privileged_fi configure_proxy(mappings={privileged_field: f"x-{privileged_field}"}) request = _request_with_headers({f"x-{privileged_field}": "proxy_admin"}) - with pytest.raises(ValueError) as exc: + with pytest.raises(ValueError, match='proxy auth refuses to map non-identity UserAPIKeyAuth') as exc: await handle_oauth2_proxy_request(request) assert privileged_field in str(exc.value) diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 6d6e20e9c36..636c5480d67 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -42,7 +42,7 @@ def test_non_admin_config_update_route_rejected(): request.query_params = {} # Test that calling /config/update route raises HTTPException with 403 status - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -134,7 +134,7 @@ def test_user_banner_update_rejected_for_non_admin(): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -1814,7 +1814,7 @@ def test_internal_user_blocked_from_global_spend_routes(route): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -1843,7 +1843,7 @@ def test_internal_user_view_only_blocked_from_global_spend_routes(route): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, @@ -2046,7 +2046,7 @@ def test_internal_user_blocked_from_admin_viewer_logs_routes(route): if route not in INTERNAL_USER_BLOCKED_SUBSET: return - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -2530,7 +2530,7 @@ def test_non_admin_non_team_admin_cannot_access_config_update_but_can_attempt_re ) # /config/update is still blocked - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -3188,7 +3188,7 @@ def test_internal_user_blocked_from_search_tool_writes(route): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index ab7e3d9701c..043bbb5b76a 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -5337,7 +5337,7 @@ async def test_random_non_sk_token_is_rejected(monkeypatch): patch("litellm.proxy.proxy_server.master_key", "sk-master"), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='LiteLLM Virtual Key expected\\.') as exc_info: await user_api_key_auth( request=mock_request, api_key="Bearer not-a-real-token", @@ -5539,7 +5539,7 @@ async def test_real_jwt_still_requires_license_when_jwt_auth_enabled(monkeypatch patch("litellm.proxy.proxy_server.master_key", "sk-master"), patch("litellm.proxy.proxy_server.prisma_client", None), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='JWT Auth is an enterprise only feature\\. You must be a') as exc_info: await user_api_key_auth( request=mock_request, api_key=f"Bearer {jwt_token}", diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py index eb40f54a1f3..be29269fe25 100644 --- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py @@ -84,7 +84,7 @@ class TestPollingErrorSurfacing: } with patch("requests.get", return_value=mock_response) as mock_get, patch("time.sleep"): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Your litellm CLI is out of date and uses a login flow') as exc_info: _poll_for_ready_data("http://test/sso/cli/poll/sk-legacy") assert mock_get.call_count == 1 @@ -151,7 +151,7 @@ class TestStartCliSsoFlowErrors: mock_response.status_code = 404 with patch("requests.post", return_value=mock_response): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Either --base-url is wrong, or the proxy is older than') as exc_info: _start_cli_sso_flow("https://old-proxy.example.com") message = str(exc_info.value) @@ -167,7 +167,7 @@ class TestStartCliSsoFlowErrors: mock_response.json.return_value = {"detail": "Too many CLI login attempts. Try again later."} with patch("requests.post", return_value=mock_response): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Too many CLI login attempts\\. Try again later\\.') as exc_info: _start_cli_sso_flow("https://test.example.com") assert "HTTP 429" in str(exc_info.value) @@ -183,7 +183,7 @@ class TestStartCliSsoFlowErrors: mock_response.text = "Sign in to corporate VPN" with patch("requests.post", return_value=mock_response): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='A proxy, load balancer, or auth gateway in front of') as exc_info: _start_cli_sso_flow("https://test.example.com") message = str(exc_info.value) @@ -197,7 +197,7 @@ class TestStartCliSsoFlowErrors: from litellm.proxy.client.cli.commands.auth import _start_cli_sso_flow with patch("requests.post", side_effect=requests.ConnectionError("Connection refused")): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Connection refused\\. Check that the proxy is running') as exc_info: _start_cli_sso_flow("https://unreachable.example.com") message = str(exc_info.value) diff --git a/tests/test_litellm/proxy/client/cli/test_pkce_login.py b/tests/test_litellm/proxy/client/cli/test_pkce_login.py index 70f481d5cfa..f58bd0ff412 100644 --- a/tests/test_litellm/proxy/client/cli/test_pkce_login.py +++ b/tests/test_litellm/proxy/client/cli/test_pkce_login.py @@ -622,7 +622,7 @@ def test_fresh_api_key_never_hands_out_a_rotated_key_it_could_not_save(): def save(_record): raise OSError("disk full") - with pytest.raises(OSError): + with pytest.raises(OSError, match="disk full"): _fresh(STORED, save, http, now=lambda: 999_950.0) diff --git a/tests/test_litellm/proxy/client/test_models.py b/tests/test_litellm/proxy/client/test_models.py index b2485032a37..33f963b74af 100644 --- a/tests/test_litellm/proxy/client/test_models.py +++ b/tests/test_litellm/proxy/client/test_models.py @@ -472,14 +472,14 @@ def test_get_invalid_params(): client = ModelsManagementClient(base_url="http://localhost:8000") # Test with no parameters - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Exactly one of model_id or model_name must be provided') as exc_info: client.get() assert "Exactly one of model_id or model_name must be provided" in str( exc_info.value ) # Test with both parameters - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Exactly one of model_id or model_name must be provided') as exc_info: client.get(model_id="123", model_name="gpt-4") assert "Exactly one of model_id or model_name must be provided" in str( exc_info.value diff --git a/tests/test_litellm/proxy/common_utils/test_callback_utils.py b/tests/test_litellm/proxy/common_utils/test_callback_utils.py index 515a7b27c7b..77ada4c11a9 100644 --- a/tests/test_litellm/proxy/common_utils/test_callback_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_callback_utils.py @@ -586,7 +586,7 @@ def test_initialize_callbacks_on_proxy_rejects_class_valued_entry(probe_config_p silently never run the hook. Config load must fail instead.""" entry = f"{_PROBE_MODULE_NAME}.FloorMaxTokens" - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='litellm_settings\\.callbacks entry') as exc_info: _load_callbacks([entry], probe_config_path) message = str(exc_info.value) @@ -609,7 +609,7 @@ def test_initialize_callbacks_on_proxy_rejects_non_dispatchable_values( ): entry = f"{_PROBE_MODULE_NAME}.{attribute}" - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='litellm_settings\\.callbacks entry') as exc_info: _load_callbacks([entry], probe_config_path) message = str(exc_info.value) @@ -621,7 +621,7 @@ def test_initialize_callbacks_on_proxy_rejects_non_dispatchable_values( def test_initialize_callbacks_on_proxy_rejects_class_valued_non_list_value(probe_config_path): entry = f"{_PROBE_MODULE_NAME}.FloorMaxTokens" - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='litellm_settings\\.callbacks entry') as exc_info: _load_callbacks(entry, probe_config_path) assert entry in str(exc_info.value) diff --git a/tests/test_litellm/proxy/common_utils/test_path_utils.py b/tests/test_litellm/proxy/common_utils/test_path_utils.py index c8d58fa8259..8936d910777 100644 --- a/tests/test_litellm/proxy/common_utils/test_path_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_path_utils.py @@ -42,5 +42,5 @@ class TestSafeFilename: safe_filename("..") def test_empty_rejected(self): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Empty or unsafe filename'): safe_filename("") diff --git a/tests/test_litellm/proxy/common_utils/test_timezone_utils.py b/tests/test_litellm/proxy/common_utils/test_timezone_utils.py index 7f686c53c95..dc3917cb48e 100644 --- a/tests/test_litellm/proxy/common_utils/test_timezone_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_timezone_utils.py @@ -130,16 +130,16 @@ def test_parse_budget_reset_time_unset_defaults_to_midnight(): def test_parse_budget_reset_time_invalid_string_raises(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="hour 'HH:MM' or 'HH:MM:SS' string, e\\.g\\."): parse_budget_reset_time("25:00") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Invalid budget_reset_time 'noon'; expected a"): parse_budget_reset_time("noon") def test_parse_budget_reset_time_non_string_raises(): # Unquoted "12:00" in YAML parses to the int 720; it must fail loudly, # not silently fall back to midnight. - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="hour 'HH:MM' string, e\\.g\\."): parse_budget_reset_time(720) diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_logs_partition_manager.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_logs_partition_manager.py index e949afce57b..4ea655b8871 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_logs_partition_manager.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_spend_logs_partition_manager.py @@ -308,9 +308,9 @@ async def test_partition_maintenance_issues_nothing_when_the_budget_is_already_s def test_unsupported_interval_raises(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Unsupported partition interval: year'): period_start(date(2026, 6, 1), "year") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Unsupported partition interval: year'): next_period_start(date(2026, 6, 1), "year") diff --git a/tests/test_litellm/proxy/db/test_prisma_planned_engine_restart.py b/tests/test_litellm/proxy/db/test_prisma_planned_engine_restart.py index c8e0338eeaa..95e794012ec 100644 --- a/tests/test_litellm/proxy/db/test_prisma_planned_engine_restart.py +++ b/tests/test_litellm/proxy/db/test_prisma_planned_engine_restart.py @@ -913,7 +913,7 @@ async def test_health_check_alerts_for_non_connection_errors_during_a_replacemen await _yield_to_loop() assert wrapper._reconnection_lock.locked() is True - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='malformed SELECT'): await client.health_check() gate.set() diff --git a/tests/test_litellm/proxy/db/test_spend_log_tool_index.py b/tests/test_litellm/proxy/db/test_spend_log_tool_index.py index 71073fd216e..9c4fbbf41aa 100644 --- a/tests/test_litellm/proxy/db/test_spend_log_tool_index.py +++ b/tests/test_litellm/proxy/db/test_spend_log_tool_index.py @@ -327,7 +327,7 @@ class TestFlushToolUsageTransactions: async def test_non_connection_errors_do_not_retry(self): prisma = MagicMock() prisma.db.batch_ = MagicMock(side_effect=ValueError("bad data")) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="bad data"): await flush_tool_usage_transactions(prisma_client=prisma, transactions=[_transaction("r1")]) prisma.db.batch_.assert_called_once() diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py index 3adf8b8407d..ceb59571389 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py @@ -56,7 +56,7 @@ def _patched(guardrail: BedrockGuardrail, http_response): def test_init_rejects_both_identifier_and_checks(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Bedrock guardrail accepts either'): BedrockGuardrail(guardrailIdentifier="gid", checks=CONTENT_FILTER_CHECKS) @@ -304,7 +304,7 @@ async def test_truncated_pii_ignored_when_pii_check_not_configured(): @pytest.mark.asyncio async def test_checks_with_guardrail_version_rejected(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Bedrock guardrail accepts either'): BedrockGuardrail(checks=CONTENT_FILTER_CHECKS, guardrailVersion="DRAFT") diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_enkryptai.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_enkryptai.py index e6c94a4c3cd..d31f462a185 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_enkryptai.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_enkryptai.py @@ -178,7 +178,7 @@ class TestEnkryptAIGuardrailHooks: with patch.object( enkryptai_guardrail.async_handler, "post", return_value=mock_response ): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='violation\\(s\\) detected') as exc_info: await enkryptai_guardrail.async_pre_call_hook( user_api_key_dict=mock_user_api_key_dict, cache=MagicMock(), diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py index 5be0d43c250..523ec1a37b4 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py @@ -767,7 +767,7 @@ class TestErrorHandling: "API Error", request=MagicMock(), response=MagicMock(status_code=500) ), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Generic Guardrail API failed: API Error') as exc_info: await generic_guardrail.apply_guardrail( inputs={"texts": ["test"]}, request_data=mock_request_data_input, @@ -786,7 +786,7 @@ class TestErrorHandling: "post", side_effect=httpx.RequestError("Connection failed", request=MagicMock()), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Generic Guardrail API failed: Connection failed') as exc_info: await generic_guardrail.apply_guardrail( inputs={"texts": ["test"]}, request_data=mock_request_data_input, @@ -810,7 +810,7 @@ class TestErrorHandling: "post", side_effect=httpx.RequestError("Connection failed", request=MagicMock()), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Generic Guardrail API failed: Connection failed') as exc_info: await guardrail.apply_guardrail( inputs={"texts": ["test"]}, request_data=mock_request_data_input, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py index 89b6af27719..14c0d2f9435 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py @@ -2334,7 +2334,7 @@ async def test_async_moderation_hook_api_error_fail_on_error_true(): } # Should raise the exception since fail_on_error is True - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="API Error") as exc_info: await guardrail.async_moderation_hook( data=request_data, user_api_key_dict=mock_user_api_key_dict, @@ -2374,7 +2374,7 @@ async def test_async_moderation_hook_api_error_fail_on_error_false(): # Even with fail_on_error=False, the decorator may still raise the exception # This test verifies that the exception is properly logged and handled - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="API Error") as exc_info: await guardrail.async_moderation_hook( data=request_data, user_api_key_dict=mock_user_api_key_dict, @@ -2865,7 +2865,7 @@ async def test_skip_unscannable_still_fails_closed_on_api_error(): "post", AsyncMock(side_effect=Exception("model armor upstream 500")), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='model armor upstream') as exc_info: await guardrail.async_pre_call_hook( user_api_key_dict=UserAPIKeyAuth(), cache=MagicMock(spec=DualCache), diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py index 2284f2b678a..8f29ba66814 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py @@ -5652,7 +5652,7 @@ class TestPanwAirsTimeoutCoercion: assert isinstance(params.timeout, float) def test_litellm_params_rejects_garbage_timeout(self): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='validation error for LitellmParams'): LitellmParams( guardrail="panw_prisma_airs", mode="pre_call", diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py index a3d86034f70..81604e22c87 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py @@ -90,12 +90,12 @@ def test_config_model_wiring(): def test_init_rejects_empty_api_key(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='api_key must be non-empty'): StraikerGuardrail(api_key="") def test_init_rejects_invalid_fallback(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="unreachable_fallback must be 'fail_open' or 'fail_closed';"): StraikerGuardrail(api_key="k", unreachable_fallback="nope") @@ -109,7 +109,7 @@ def test_supported_hooks_limited_to_pre_and_post(): def test_during_call_mode_rejected_at_init(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Event hook GuardrailEventHooks\\.during_call is not in the'): StraikerGuardrail(api_key="k", event_hook="during_call") diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py index 4b381b67f0e..0c5addbc143 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py @@ -124,7 +124,7 @@ class TestToolPermissionGuardrail: assert rule_id is None def test_rule_requires_name_or_type(self): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='validation error for ToolPermissionRule'): ToolPermissionGuardrail( guardrail_name="invalid-rule", rules=[{"id": "no_target", "decision": "allow"}], @@ -1042,7 +1042,7 @@ class TestToolPermissionGuardrailInMemoryUpdate: assert guardrail._check_tool_permission("Secret")[0] is False assert guardrail._check_tool_permission("Other")[0] is True - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Invalid regex for tool_name in rule 'bad': unterminated"): guardrail.update_in_memory_litellm_params( LitellmParams( guardrail="tool_permission", diff --git a/tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py b/tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py index 45dec4ddb2d..bd2553b3280 100644 --- a/tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py +++ b/tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py @@ -230,7 +230,7 @@ def test_parse_judge_verdict_reraises_when_no_json(): def test_parse_judge_verdict_rejects_json_non_object(): """Valid JSON that is not an object (e.g. a bare list) raises ValueError.""" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='judge response is not a JSON object'): _parse_judge_verdict("[1, 2, 3]") diff --git a/tests/test_litellm/proxy/hooks/test_sensitive_data_routing.py b/tests/test_litellm/proxy/hooks/test_sensitive_data_routing.py index 78d2c3af0f3..35c0f8deaf1 100644 --- a/tests/test_litellm/proxy/hooks/test_sensitive_data_routing.py +++ b/tests/test_litellm/proxy/hooks/test_sensitive_data_routing.py @@ -227,7 +227,7 @@ class TestCustomGuardrailSensitiveDataRouting: request_data = {"model": "gpt-4"} - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Cannot route sensitive data without a session_id\\. Ensure') as exc_info: guardrail.raise_sensitive_data_route_exception( route_to_model="on-premise-model", request_data=request_data, diff --git a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py index 8d03857c917..2839acab6b0 100644 --- a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py +++ b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py @@ -150,7 +150,7 @@ async def test_no_leak_on_over_limit_rejection(rate_limiter): f"estimated={estimated}, limit={user_api_key_dict.tpm_limit}" ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Limit type: tokens\\. Current limit') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -685,7 +685,7 @@ async def test_contentless_request_reserves_minimum(rate_limiter): f"counter should be 2, got {counter_after_two}" ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Limit type: tokens\\. Current limit') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -1319,7 +1319,7 @@ async def test_project_otpm_rejects_multiple_completion_candidates(rate_limiter) "n": 10, } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -1347,7 +1347,7 @@ async def test_project_otpm_reserves_largest_conflicting_output_cap(rate_limiter "max_completion_tokens": 100, } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -1377,7 +1377,7 @@ async def test_project_otpm_rejects_google_genai_native_output_cap( project_metadata={"model_otpm_limit": {model: 50}}, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -1411,7 +1411,7 @@ async def test_project_otpm_rejects_google_genai_native_candidate_count( project_metadata={"model_otpm_limit": {model: 150}}, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -1500,7 +1500,7 @@ async def test_project_otpm_over_limit_rolls_back_itpm_reservation(rate_limiter) "max_tokens": 500, # blows past the 10-token OTPM limit } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -2003,7 +2003,7 @@ async def test_otpm_rejection_does_not_double_refund_combined_tpm(rate_limiter): rate_limit_type="tokens", ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -2060,7 +2060,7 @@ async def test_project_itpm_rejects_pretokenized_embedding_input( "input": embedding_input, } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_itpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -2250,7 +2250,7 @@ async def test_itpm_reservation_accounts_for_audio_content_not_just_text(rate_li ], } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_itpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -2361,7 +2361,7 @@ async def test_itpm_rejects_large_audio_payload_that_would_pass_flat_estimate( ], } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_itpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -2622,7 +2622,7 @@ async def test_explicit_zero_output_responses_call_reserves_effective_provider_m }, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, @@ -2850,7 +2850,7 @@ async def test_otpm_rejection_releases_stashed_parallel_slot(rate_limiter): "rate_limit": {"tokens_per_unit": 5, "window_size": 60}, } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_otpm') as exc_info: await handler._reserve_project_io_tokens_or_raise( descriptors=[otpm_descriptor], data=data, @@ -3296,7 +3296,7 @@ async def test_rerank_query_and_documents_enforce_project_itpm( project_metadata={"model_itpm_limit": {"rerank-model": 100}}, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Rate limit exceeded for model_per_project_itpm') as exc_info: await handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=cache, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index d4e9ccdca5e..069cfa01178 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -10179,7 +10179,7 @@ async def test_update_key_creator_reassigned_key_blocked(monkeypatch): mock_request = MagicMock() mock_request.query_params = {} - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='User can only create keys for themselves\\. Got') as exc: await update_key_fn( request=mock_request, data=UpdateKeyRequest(key=test_hashed_token, key_alias="hijacked"), diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index 84dee5b05c5..01c0760bd27 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -2325,7 +2325,7 @@ class TestTemporaryMCPSessionEndpoints: "litellm.proxy.management_endpoints.mcp_management_endpoints.validate_and_normalize_mcp_server_payload", MagicMock(), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='User does not have permission to create temporary mcp') as exc_info: await add_session_mcp_server( payload=payload, user_api_key_dict=non_admin, diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 7e4596d154b..42e96ad8659 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -140,7 +140,7 @@ class TestModelManagementAuthChecks: @pytest.mark.asyncio async def test_can_user_make_team_model_call_non_premium_fails(self): """Test that non-premium users cannot make team model calls""" - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='You must be a LiteLLM Enterprise user to use this feature\\.') as exc_info: ModelManagementAuthChecks.can_user_make_team_model_call( team_id="test_team", user_api_key_dict=self.admin_user, @@ -195,7 +195,7 @@ class TestModelManagementAuthChecks: ) prisma_client = MockPrismaClient(team_exists=True) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='You must be a LiteLLM Enterprise user to use this feature\\.') as exc_info: await ModelManagementAuthChecks.allow_team_model_action( model_params=model_params, user_api_key_dict=self.admin_user, @@ -216,7 +216,7 @@ class TestModelManagementAuthChecks: ) prisma_client = MockPrismaClient(team_exists=False) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Team id=nonexistent_team does not exist in db'\\}") as exc_info: await ModelManagementAuthChecks.allow_team_model_action( model_params=model_params, user_api_key_dict=self.admin_user, @@ -257,7 +257,7 @@ class TestModelManagementAuthChecks: ) prisma_client = MockPrismaClient(team_exists=True, user_admin=False) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Team ID=test_team does not match the API key's team") as exc_info: await ModelManagementAuthChecks.can_user_make_model_call( model_params=model_params, user_api_key_dict=self.normal_user, @@ -1483,7 +1483,7 @@ class TestTeamModelUpdate: "litellm.proxy.proxy_server.premium_user", True, ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="does not match the API key's team ID=None, OR you are") as exc_info: await _update_team_model_in_db( db_model=db_model, patch_data=patch_data, @@ -3256,7 +3256,7 @@ class TestPatchModelBlockedAuthGate: new=AsyncMock(return_value=None), ), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Only proxy admins can change a model's blocked flag\\.") as exc_info: await patch_model( model_id="m1", patch_data=updateDeployment(blocked=True), diff --git a/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py b/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py index 92a34b5ee7c..a1c38d26b9d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py @@ -58,7 +58,7 @@ def test_model_info_accepts_valid_ptu_fields(): def test_model_info_rejects_non_positive_count(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='value_error, input_value'): ModelInfo( id="x", team_id="t", @@ -69,7 +69,7 @@ def test_model_info_rejects_non_positive_count(): def test_model_info_rejects_negative_rate(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='value_error, input_value'): ModelInfo( id="x", team_id="t", @@ -82,7 +82,7 @@ def test_model_info_rejects_negative_rate(): def test_model_info_rejects_a_count_beyond_the_cap(): """flat cost multiplies the count by a float, and an unbounded int overflows that conversion, which aborted the rollup for every team rather than skipping one model.""" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='validation error for ModelInfo'): ModelInfo(id="x", team_id="t", ptu_count=10**400, cost_per_ptu_per_hour=2.0) @@ -95,12 +95,12 @@ def test_model_info_accepts_a_count_at_the_cap(): def test_model_info_rejects_a_non_finite_rate(rate): """NaN compares False against every bound, so a bare `< 0` check let it through and the deployment then accrued a flat cost of nan.""" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='value_error, input_value'): ModelInfo(id="x", team_id="t", ptu_count=5, cost_per_ptu_per_hour=rate) def test_model_info_rejects_a_rate_beyond_the_cap(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='validation error for ModelInfo'): ModelInfo(id="x", team_id="t", ptu_count=5, cost_per_ptu_per_hour=ModelInfo.MAX_COST_PER_PTU_PER_HOUR * 2) @@ -148,7 +148,7 @@ def test_validate_helper_passes_full_config(): def test_model_info_rejects_effective_to_before_from(): import datetime - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='validation error for ModelInfo'): ModelInfo( id="x", team_id="t", @@ -186,7 +186,7 @@ def test_model_info_compares_mixed_naive_and_aware_timestamps(): ) assert info.ptu_effective_to is not None - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='validation error for ModelInfo'): ModelInfo( id="x", team_id="t", @@ -698,7 +698,7 @@ class TestAddNewModelPtuGate: with ExitStack() as stack: for active_patch in patches: stack.enter_context(active_patch) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='PTU cost attribution is disabled, so ptu_count') as exc: await add_new_model(model_params=self._ptu_deployment("ptu-gate-model"), user_api_key_dict=admin) assert PTU_COST_ATTRIBUTION_ENV_VAR in str(exc.value) @@ -1273,7 +1273,7 @@ class TestPtuDeploymentsAreNotBilledPerToken: with ExitStack() as stack: for active_patch in patches: stack.enter_context(active_patch) - with pytest.raises(Exception) as exc: + with pytest.raises(Exception, match='A PTU deployment bills by reserved capacity, so') as exc: await add_new_model(model_params=deployment, user_api_key_dict=admin) assert "input_cost_per_token" in str(exc.value) diff --git a/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py b/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py index e8a74e41dae..3a32b3cc128 100644 --- a/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py +++ b/tests/test_litellm/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py @@ -458,7 +458,7 @@ class TestUsageAiChatServiceAccountGuard: _resolve_fetch_kwargs, ) - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Non-admin caller has user_id=None; refusing to issue an') as exc_info: _resolve_fetch_kwargs( fn_name="get_usage_data", fn_args={"start_date": "2025-01-01", "end_date": "2025-01-31"}, diff --git a/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py b/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py index e4b031ade57..1acb8e7e016 100644 --- a/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py +++ b/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py @@ -671,7 +671,7 @@ async def test_non_callable_validator_is_rejected_with_clean_500(): def test_parse_schema_duplicate_error_lists_offending_keys(): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='team_metadata_schema contains duplicate keys: app_name') as exc_info: parse_team_metadata_schema( [{"key": "cost_center"}, {"key": "app_name"}, {"key": "cost_center"}, {"key": "app_name"}] ) diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_batch_guardrails.py b/tests/test_litellm/proxy/openai_files_endpoint/test_batch_guardrails.py index a05b8ae530c..6ce7af1e2ee 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_batch_guardrails.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_batch_guardrails.py @@ -781,7 +781,7 @@ async def test_the_rewrite_closes_its_own_output_when_it_cannot_finish(): original_read = bg._read_spooled bg._read_spooled = _boom try: - with pytest.raises(OSError): + with pytest.raises(OSError, match='no space left on device'): rewrite_batch_input_file(source, result) finally: bg.tempfile.SpooledTemporaryFile = real diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index f994fba371b..d3237f5f49d 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -2714,7 +2714,7 @@ class TestMilvusProxyRoute: None ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Vector store not found for missing-store') as exc_info: await milvus_proxy_route( endpoint="vectors/search", request=mock_request, @@ -2779,7 +2779,7 @@ class TestMilvusProxyRoute: mock_vector_store ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='api_base not found in vector store configuration for') as exc_info: await milvus_proxy_route( endpoint="vectors/search", request=mock_request, @@ -2988,7 +2988,7 @@ class TestOpenAIPassthroughRoute: "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials", return_value=None, ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="Required 'OPENAI_API_KEY' in environment to make") as exc_info: await openai_proxy_route( endpoint="v1/chat/completions", request=mock_request, @@ -3177,7 +3177,7 @@ class TestCursorProxyRoute: [], ), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Cursor API key not found\\. Add Cursor credentials via') as exc_info: await cursor_proxy_route( endpoint="v0/agents", request=mock_request, diff --git a/tests/test_litellm/proxy/policy_engine/test_policy_versioning.py b/tests/test_litellm/proxy/policy_engine/test_policy_versioning.py index ebebfde5cd3..b6633779326 100644 --- a/tests/test_litellm/proxy/policy_engine/test_policy_versioning.py +++ b/tests/test_litellm/proxy/policy_engine/test_policy_versioning.py @@ -157,7 +157,7 @@ class TestUpdatePolicyDraftOnly: prod_row = _make_row(policy_id="pid-1", version_status="production") prisma.db.litellm_policytable.find_unique = AsyncMock(return_value=prod_row) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Error updating policy in DB: Only draft versions can be') as exc_info: await registry.update_policy_in_db( policy_id="pid-1", policy_request=PolicyUpdateRequest(description="new"), @@ -341,7 +341,7 @@ class TestUpdateVersionStatus: draft = _make_row(policy_id="d-1", version_status="draft") prisma.db.litellm_policytable.find_unique = AsyncMock(return_value=draft) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Error updating version status: Cannot promote draft') as exc_info: await registry.update_version_status( policy_id="d-1", new_status="production", diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 54ae279d005..47f01fe096d 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -1504,7 +1504,7 @@ async def test_ProxyConfig__init_non_llm_configs_premium_invalid_worker_registry async def test_ProxyConfig__init_non_llm_configs_worker_registry_requires_premium(monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) pc = ProxyConfig() - with pytest.raises(ValueError) as exc_info: + with pytest.raises(ValueError, match='Trying to use `worker_registry`You must be a LiteLLM') as exc_info: await pc._init_non_llm_configs( config={ "worker_registry": [ @@ -1769,7 +1769,7 @@ def test_ProxyConfig_initialize_secret_manager_none_noop(): def test_ProxyConfig_initialize_secret_manager_invalid_kms_raises(): pc = ProxyConfig() - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Invalid Key Management System selected'): pc.initialize_secret_manager(key_management_system="not-a-real-kms") diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 15a3e6609f0..b2ec500d045 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -3263,7 +3263,7 @@ async def test_provider_budget_over(disable_budget_sync): model_list=MODEL_LIST, ) - with pytest.raises(Exception) as e: + with pytest.raises(Exception, match='No deployments available - crossed budget: Exceeded budget') as e: await router.acompletion( model="azure-gpt-4o", messages=[{"role": "user", "content": "Hello, world!"}], @@ -5096,7 +5096,7 @@ def test_resolve_spend_report_scope_missing_caller_value_400(): @pytest.mark.parametrize("bad_column", ["metadata", "end_user", "evil; DROP TABLE", ""]) def test_scoped_spend_report_sql_rejects_unknown_column(bad_column): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Unsupported spend report scope column'): spend_management_endpoints._scoped_spend_report_sql(scope_column=bad_column) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index c693f5ab2cb..716fba370df 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -435,7 +435,7 @@ class TestProxyBaseLLMRequestProcessing: # Test with invalid header value (should raise ValueError when converting to float) headers_with_invalid = {"x-litellm-stream-timeout": "invalid"} - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="could not convert string to float: 'invalid"): LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers_with_invalid) @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/test_enforce_user_param.py b/tests/test_litellm/proxy/test_enforce_user_param.py index 6891123e70e..1001372aeb5 100644 --- a/tests/test_litellm/proxy/test_enforce_user_param.py +++ b/tests/test_litellm/proxy/test_enforce_user_param.py @@ -56,7 +56,7 @@ class TestEnforceUserParamPostGetFiltering: new_callable=AsyncMock, return_value=True, ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="user' param not passed in\\. 'enforce_user_param'=True") as exc_info: await common_checks( request_body=request_body, team_object=None, @@ -175,7 +175,7 @@ class TestEnforceUserParamPostGetFiltering: new_callable=AsyncMock, return_value=True, ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="user' param not passed in\\. 'enforce_user_param'=True") as exc_info: await common_checks( request_body=request_body, team_object=None, @@ -405,7 +405,7 @@ class TestEnforceUserParamEdgeCases: new_callable=AsyncMock, return_value=True, ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="user' param not passed in\\. 'enforce_user_param'=True") as exc_info: await common_checks( request_body=request_body, team_object=None, diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index b1071150f3b..636974d5deb 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -2279,12 +2279,12 @@ def test_get_num_retries_from_request(): # Test case 7: Header present with invalid value (should raise ValueError when int() is called) headers_with_invalid = {"x-litellm-num-retries": "invalid"} - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='invalid literal for int\\(\\) with base'): LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_invalid) # Test case 8: Header present with float string (should raise ValueError when int() is called) headers_with_float = {"x-litellm-num-retries": "3.5"} - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='invalid literal for int\\(\\) with base'): LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_float) # Test case 9: Header present with negative number @@ -2324,7 +2324,7 @@ def test_get_keepalive_seconds_from_request(): # Header present with invalid value raises ValueError, matching the other # x-litellm-* numeric header helpers (_get_timeout_from_request, etc.) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="could not convert string to float: 'not-a-number"): LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request( {"x-litellm-keepalive-seconds": "not-a-number"} ) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 75aa716bb85..83e9095c8ec 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -1507,7 +1507,7 @@ def test_team_info_masking(): "langfuse_public_key": "public-test-key", } - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match="secr\\*\\*\\*\\*\\*\\*\\*-key', 'langfuse_public_key':") as exc_info: proxy_config._get_team_config( team_id="test_dev", all_teams_config=[team1_info], diff --git a/tests/test_litellm/proxy/test_spend_log_cleanup.py b/tests/test_litellm/proxy/test_spend_log_cleanup.py index ce0b6b755cc..bf1538183ab 100644 --- a/tests/test_litellm/proxy/test_spend_log_cleanup.py +++ b/tests/test_litellm/proxy/test_spend_log_cleanup.py @@ -82,10 +82,10 @@ def test_spend_log_cleanup_cron_scheduling(): assert trigger_weekly is not None # Invalid cron expression should raise ValueError - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Wrong number of fields; got'): CronTrigger.from_crontab("invalid cron") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='is higher than the maximum value'): CronTrigger.from_crontab("60 25 * * *") # Invalid minute and hour diff --git a/tests/test_litellm/proxy/test_team_org_move.py b/tests/test_litellm/proxy/test_team_org_move.py index 2dc961bec85..064e9de550e 100644 --- a/tests/test_litellm/proxy/test_team_org_move.py +++ b/tests/test_litellm/proxy/test_team_org_move.py @@ -97,7 +97,7 @@ class TestValidateTeamOrgChange: team = _make_team(member_ids=["sso-user-001"]) org = _make_org(members=[]) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Cannot move team to organization\\. Team has user_id') as exc_info: validate_team_org_change( team=team, organization=org, llm_router=router, is_proxy_admin=False ) diff --git a/tests/test_litellm/proxy/utils/helpers/test_team_configs.py b/tests/test_litellm/proxy/utils/helpers/test_team_configs.py index 0e0906892b0..185d4d26ff4 100644 --- a/tests/test_litellm/proxy/utils/helpers/test_team_configs.py +++ b/tests/test_litellm/proxy/utils/helpers/test_team_configs.py @@ -66,7 +66,7 @@ def test_is_valid_team_configs_short_circuits_when_team_id_none(): def test_is_valid_team_configs_raises_on_model_not_in_team_models(): team_config = {"models": ["gpt-4o"]} request_data = {"model": "claude-haiku"} - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='claude-haiku\\. Valid models for team are') as exc_info: _is_valid_team_configs( team_id="team-1", team_config=team_config, diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py index 7057a112c83..93c99c7fd04 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_proxy_update_spend.py @@ -561,7 +561,7 @@ async def test_update_spend_logs_does_not_requeue_non_transport_failures( proxy_logging.failure_handler = AsyncMock() mock_prisma_client.spend_log_transactions = [] - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="bad payload"): await ProxyUpdateSpend.update_spend_logs( n_retry_times=1, prisma_client=mock_prisma_client, diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_callback_capabilities_class.py b/tests/test_litellm/proxy/utils/proxy_logging/test_callback_capabilities_class.py index 9452e8042bd..75a91177f00 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_callback_capabilities_class.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_callback_capabilities_class.py @@ -181,7 +181,7 @@ def test_has_streaming_callbacks_error_when_resolution_fails(monkeypatch): "get_custom_logger_compatible_class", lambda *a, **kw: (_ for _ in ()).throw(ValueError("nope")), ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="nope"): ProxyLogging.has_streaming_callbacks() diff --git a/tests/test_litellm/responses/test_responses_websocket_all_providers.py b/tests/test_litellm/responses/test_responses_websocket_all_providers.py index 2d523bfdeb3..e8333214ea8 100644 --- a/tests/test_litellm/responses/test_responses_websocket_all_providers.py +++ b/tests/test_litellm/responses/test_responses_websocket_all_providers.py @@ -84,7 +84,7 @@ class TestResponsesAPIWebSocketSupport: def test_azure_websocket_url_requires_api_base(self): config = AzureOpenAIResponsesAPIConfig() - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='api_base is required for Azure WebSocket'): config.get_websocket_url(api_base=None, litellm_params={}) def test_azure_model_not_in_websocket_url(self): diff --git a/tests/test_litellm/router_strategy/adaptive_router/test_bandit.py b/tests/test_litellm/router_strategy/adaptive_router/test_bandit.py index ab322f0fb37..78390cc1193 100644 --- a/tests/test_litellm/router_strategy/adaptive_router/test_bandit.py +++ b/tests/test_litellm/router_strategy/adaptive_router/test_bandit.py @@ -100,7 +100,7 @@ def test_score_combines_quality_and_cost(): def test_pick_best_empty_dict_raises(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='pick_best called with no models'): pick_best({}, {}) diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py index 73491490b14..60b1166de73 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py @@ -656,7 +656,7 @@ async def test_negation_all_excluded_raises(): enable_tag_filtering=True, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -699,7 +699,7 @@ async def test_negation_ban_only_cannot_escape_default_pool(): # Sending only "!default" must NOT route to the paid deployment. # The base pool for ban-only is the default pool; banning the only # default deployment should raise rather than falling through to paid. - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -969,7 +969,7 @@ async def test_negation_exhausts_entire_fallback_chain(): enable_tag_filtering=True, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="primary", messages=[{"role": "user", "content": "hi"}], @@ -1719,7 +1719,7 @@ async def test_required_and_unmatched_raises_by_default(): enable_tag_filtering=True, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -1751,7 +1751,7 @@ async def test_required_and_combined_with_positive_unmatched_raises_by_default() enable_tag_filtering=True, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -1973,7 +1973,7 @@ async def test_negation_combined_with_positive_unmatched_raises_by_default(): enable_tag_filtering=True, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -2131,7 +2131,7 @@ async def test_mixed_constraint_survivor_unmatched_by_positive_tag_raises_by_def enable_tag_filtering=True, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -2224,7 +2224,7 @@ async def test_allow_fail_open_denied_when_request_includes_unknown_tag(): enable_tag_filtering=True, ) - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -2538,7 +2538,7 @@ async def test_plain_tag_exhaustion_with_universal_default_tag_raises_by_default "litellm.router._async_get_cooldown_deployments", new=AsyncMock(return_value=["quality-high-1", "quality-high-2"]), ): - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gpt-4", messages=[{"role": "user", "content": "hi"}], @@ -2767,7 +2767,7 @@ async def test_allow_fail_open_raises_when_inherited_constraint_alone_is_unsatis # allow_fail_open unset. router = _eu_region_router() - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="chat", messages=[{"role": "user", "content": "hi"}], @@ -2941,7 +2941,7 @@ async def test_tagged_request_direct_to_plain_group_still_rejected(): # tag filtering must reject exactly as before. router = _tagged_marker_router() - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gemini-flash", messages=[{"role": "user", "content": "hi"}], @@ -2962,7 +2962,7 @@ async def test_caller_forged_consumption_stamp_is_neutralized_by_the_hook(): # tag filtering runs. router = _tagged_marker_router() - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await router.acompletion( model="gemini-flash", messages=[{"role": "user", "content": "hi"}], @@ -2984,7 +2984,7 @@ async def test_inherited_constraint_still_applies_to_the_routed_tier(): # ®ion:eu comes from key/team policy (present in inherited_tags): # consuming the router-selecting "route" tag must not also discard the # inherited requirement, so a tier without the tag still raises... - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: await _tagged_marker_router().acompletion( model="gpt4o", messages=[{"role": "user", "content": "hi"}], diff --git a/tests/test_litellm/sandbox/test_e2b_sandbox.py b/tests/test_litellm/sandbox/test_e2b_sandbox.py index cc5b12156a1..e01b9120416 100644 --- a/tests/test_litellm/sandbox/test_e2b_sandbox.py +++ b/tests/test_litellm/sandbox/test_e2b_sandbox.py @@ -293,7 +293,7 @@ async def test_public_lifecycle_create_run_delete(): @pytest.mark.asyncio async def test_unsupported_provider_raises(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="not-a-provider' is not a valid SandboxProviders"): await litellm.acreate_sandbox(provider="not-a-provider") diff --git a/tests/test_litellm/secret_managers/test_base_secret_manager.py b/tests/test_litellm/secret_managers/test_base_secret_manager.py index cba6a99ab7f..e1ccb91c381 100644 --- a/tests/test_litellm/secret_managers/test_base_secret_manager.py +++ b/tests/test_litellm/secret_managers/test_base_secret_manager.py @@ -32,7 +32,7 @@ from litellm.secret_managers.base_secret_manager import raise_if_unsafe_secret_n ], ) def test_raise_if_unsafe_secret_name_rejects_traversal_and_line_breaks(secret_name): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Invalid secret_name'): raise_if_unsafe_secret_name(secret_name) diff --git a/tests/test_litellm/test_github_close_low_quality_prs.py b/tests/test_litellm/test_github_close_low_quality_prs.py index e3b653dde64..2a891ca72f5 100644 --- a/tests/test_litellm/test_github_close_low_quality_prs.py +++ b/tests/test_litellm/test_github_close_low_quality_prs.py @@ -697,7 +697,7 @@ class TestListOpenItemsNoCap: def test_list_open_items_rejects_unknown_kind(self, closer_module): shared = self._shared(closer_module) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="kind must be 'pr' or 'issue', got 'both"): shared.list_open_items("both", repo="o/r", fields="number") def test_fetch_open_prs_delegates_with_no_cap(self, closer_module, monkeypatch): diff --git a/tests/test_litellm/test_github_triage_with_llm.py b/tests/test_litellm/test_github_triage_with_llm.py index 96b77e80457..ddffb978b48 100644 --- a/tests/test_litellm/test_github_triage_with_llm.py +++ b/tests/test_litellm/test_github_triage_with_llm.py @@ -665,11 +665,11 @@ class TestParseVerdict: assert triage_module.parse_verdict(raw)["verdict"] == "pass" def test_should_raise_for_unparseable_text(self, triage_module): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='could not extract JSON from LLM response: not even close to'): triage_module.parse_verdict("not even close to json") def test_should_raise_for_empty(self, triage_module): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='empty LLM response'): triage_module.parse_verdict("") diff --git a/tests/test_litellm/test_redact_string_in_error_paths.py b/tests/test_litellm/test_redact_string_in_error_paths.py index 8eddc2b1a5a..1c4d91397d1 100644 --- a/tests/test_litellm/test_redact_string_in_error_paths.py +++ b/tests/test_litellm/test_redact_string_in_error_paths.py @@ -234,7 +234,7 @@ class TestRouterFallbackFailureTracebackRedaction: raise ValueError(f"primary deployment failed api_key={secret}") except ValueError as original_exception: with caplog.at_level(logging.DEBUG, logger="LiteLLM Router"): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='primary deployment failed api_key=sk-testsecretvalu'): await router.async_function_with_fallbacks_common_utils( e=original_exception, disable_fallbacks=False, diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index 3aa4bc58f13..c645a67ef84 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -131,7 +131,7 @@ def test_get_redis_url_from_environment_missing_host_port(monkeypatch): monkeypatch.delenv("REDIS_PORT", raising=False) # Call the function and expect a ValueError - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match="Either 'REDIS_URL' or both 'REDIS_HOST' and 'REDIS_PORT") as excinfo: get_redis_url_from_environment() # Check the error message @@ -149,7 +149,7 @@ def test_get_redis_url_from_environment_missing_port(monkeypatch): monkeypatch.setenv("REDIS_HOST", "redis-server") # Call the function and expect a ValueError - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match="Either 'REDIS_URL' or both 'REDIS_HOST' and 'REDIS_PORT") as excinfo: get_redis_url_from_environment() # Check the error message diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index b50dc92c220..a47525749f6 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -953,7 +953,7 @@ async def test_arouter_filter_team_based_models(): assert result is not None # FAILS - with pytest.raises(Exception) as e: + with pytest.raises(Exception, match='No deployments available for selected model, Try again in') as e: result = await router.acompletion( model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hello, world!"}], @@ -1225,7 +1225,7 @@ def test_add_invalid_provider_to_router(): ], ) - with pytest.raises(Exception) as e: + with pytest.raises(Exception, match='Unsupported provider - vertex_ai_eu') as e: router.add_deployment( Deployment( model_name="vertex_ai/*", @@ -1320,7 +1320,7 @@ async def test_router_ageneric_api_call_with_fallbacks_helper(): with patch.object(router, "async_get_available_deployment") as mock_get_deployment: mock_get_deployment.side_effect = Exception("No deployment available") - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='No deployment available') as exc_info: await router._ageneric_api_call_with_fallbacks_helper( model="gpt-3.5-turbo", original_generic_function=mock_generic_function, @@ -1394,7 +1394,7 @@ async def test_router_ageneric_api_call_with_fallbacks_helper(): with patch.object( router, "async_routing_strategy_pre_call_checks" ) as mock_pre_call_checks: - with pytest.raises(Exception) as exc_info: + with pytest.raises(Exception, match='Mock failure') as exc_info: await router._ageneric_api_call_with_fallbacks_helper( model="gpt-3.5-turbo", original_generic_function=mock_failing_function, @@ -3737,7 +3737,7 @@ def test_count_pre_call_check_tokens_across_api_surfaces(): assert string_input_tokens > 0 assert list_input_tokens > 0 - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='Either messages or input must be provided to count tokens'): router._count_pre_call_check_tokens(messages=None, input=None) diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 5bb854c12e0..dc210f900bf 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -1706,7 +1706,7 @@ def test_an_incomplete_reservation_is_refused_rather_than_served(dropped): state the operator was trying to leave.""" incomplete = {k: v for k, v in _PTU_MODEL_INFO.items() if k != dropped} - with pytest.raises(ValueError) as raised: + with pytest.raises(ValueError, match="PTU configuration on model 'gpt") as raised: _ptu_router(model_info=incomplete, litellm_params={"input_cost_per_token": 5e-06}) assert "gpt-4o-ptu" in str(raised.value) @@ -1726,7 +1726,7 @@ def test_the_refusal_reason_is_the_one_the_model_endpoint_answers_with(dropped, incomplete = {k: v for k, v in _PTU_MODEL_INFO.items() if k != dropped} assert ptu_config_error(incomplete) == expected - with pytest.raises(ValueError) as raised: + with pytest.raises(ValueError, match="PTU configuration on model 'gpt") as raised: _ptu_router(model_info=incomplete) assert expected in str(raised.value) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 041c60e0ba6..075b455e4b5 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4370,7 +4370,7 @@ class TestVertexEmbeddingEncodingFormat: assert "encoding_format" not in optional_params def test_encoding_format_base64_still_rejected_without_drop_params(self): - with pytest.raises(Exception) as excinfo: + with pytest.raises(Exception, match='To drop these, set `litellm\\.drop_params=True` or for proxy') as excinfo: litellm.utils.get_optional_params_embeddings( model="gemini-embedding-001", encoding_format="base64", diff --git a/tests/test_litellm/types/test_router.py b/tests/test_litellm/types/test_router.py index 5ce5eca4954..accd3b32a0d 100644 --- a/tests/test_litellm/types/test_router.py +++ b/tests/test_litellm/types/test_router.py @@ -87,5 +87,5 @@ def test_pricing_strings_are_coerced_to_float(): def test_invalid_pricing_is_rejected(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='validation error for ModelInfo'): ModelInfo(id="x", input_cost_per_token="free") From 9697748f92d1a02c4b2cfc2768c20c39242503d4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 22:28:43 -0700 Subject: [PATCH 09/12] test: gate the already-hashed pass-through on the provenance flag The spend-log helper no longer treats a 64-hex shape as proof a value was already hashed, so this case has to say where the hash came from. Reconciles the test that came in with #31799 against that change. --- .../proxy/spend_tracking/test_spend_tracking_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 706466d5033..e34d4f389e1 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2927,7 +2927,7 @@ class TestSpendLogKeyRedaction: def test_already_hashed_key_unchanged(self): hashed = "bcfe8173f5447f10be0e7fb37aaa8b97829d5c9e0498232152f9d123456789ab" - assert _redact_logged_api_key(hashed) == hashed + assert _redact_logged_api_key(hashed, already_hashed=True) == hashed def test_bearer_prefixed_non_sk_key_is_hashed(self): raw = "Bearer some-other-token-format" From fb417a556300fd6a983af66e28e0ee759d41a041 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 22:38:04 -0700 Subject: [PATCH 10/12] fix(spend-tracking): tie the already-hashed pass-through to provenance The hashed-jwt branch trusted the value's shape alone, so a caller-supplied key in that shape was stored unhashed. Both pass-throughs now require the value to match the auth-time user_api_key_hash, and the shape check is a full match. --- .../spend_tracking/spend_tracking_utils.py | 14 ++++++----- .../test_spend_tracking_utils.py | 25 +++++++++++++++++-- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 4592add1032..692200b856c 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -64,7 +64,11 @@ def _is_master_key(api_key: str | None, _master_key: str | None) -> bool: return secrets.compare_digest(api_key, _master_key) -_HASHED_JWT_RE = re.compile(r"^hashed-jwt-[a-fA-F0-9]{64}$") +_HASHED_JWT_RE = re.compile(r"hashed-jwt-[a-fA-F0-9]{64}") + + +def _is_prehashed_key_shape(value: str) -> bool: + return is_valid_sha256_hash(value) or _HASHED_JWT_RE.fullmatch(value) is not None def _redact_logged_api_key(value: str | None, *, already_hashed: bool = False) -> str | None: @@ -73,9 +77,7 @@ def _redact_logged_api_key(value: str | None, *, already_hashed: bool = False) - stripped: Final = re.sub(r"(?i)^bearer ", "", value) if not stripped: return None - if already_hashed and is_valid_sha256_hash(stripped): - return stripped - if _HASHED_JWT_RE.match(stripped): + if already_hashed and _is_prehashed_key_shape(stripped): return stripped return hash_token(stripped) @@ -136,7 +138,7 @@ def _get_spend_logs_metadata( _raw_key: Final = clean_metadata.get("user_api_key") _trusted_hash: Final = metadata.get("user_api_key_hash") _already_hashed: Final = ( - isinstance(_trusted_hash, str) and is_valid_sha256_hash(_trusted_hash) and _trusted_hash == _raw_key + isinstance(_trusted_hash, str) and _is_prehashed_key_shape(_trusted_hash) and _trusted_hash == _raw_key ) clean_metadata["user_api_key"] = _redact_logged_api_key(_raw_key, already_hashed=_already_hashed) clean_metadata["applied_guardrails"] = applied_guardrails @@ -296,7 +298,7 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs standard_logging_total_tokens = standard_logging_payload.get("total_tokens", 0) _trusted_hash = metadata.get("user_api_key_hash") _key_already_hashed = ( - isinstance(_trusted_hash, str) and is_valid_sha256_hash(_trusted_hash) and _trusted_hash == api_key + isinstance(_trusted_hash, str) and _is_prehashed_key_shape(_trusted_hash) and _trusted_hash == api_key ) api_key = _redact_logged_api_key(api_key, already_hashed=_key_already_hashed) or "" diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index e34d4f389e1..92e78e04512 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2653,10 +2653,24 @@ def test_redact_logged_api_key_long_opaque_token_is_hashed(): def test_redact_logged_api_key_hashed_jwt_passes_through(): jwt_hash = "hashed-jwt-" + "a" * 64 - result = _redact_logged_api_key(jwt_hash) + result = _redact_logged_api_key(jwt_hash, already_hashed=True) assert result == jwt_hash +def test_redact_logged_api_key_hashed_jwt_shape_without_provenance_is_hashed(): + lookalike = "hashed-jwt-" + "a" * 64 + result = _redact_logged_api_key(lookalike) + assert result == hash_token(lookalike) + assert result != lookalike + + +def test_redact_logged_api_key_hashed_jwt_trailing_newline_is_hashed(): + trailing = "hashed-jwt-" + "a" * 64 + "\n" + result = _redact_logged_api_key(trailing, already_hashed=True) + assert result == hash_token(trailing) + assert result != trailing + + def test_redact_logged_api_key_hashed_jwt_short_suffix_is_hashed(): short_jwt = "hashed-jwt-tooshort" result = _redact_logged_api_key(short_jwt) @@ -2730,10 +2744,17 @@ def test_get_spend_logs_metadata_provenance_bypass_requires_hash_match(): def test_get_spend_logs_metadata_hashed_jwt_unchanged(): jwt_hash = "hashed-jwt-" + "b" * 64 - meta = _get_spend_logs_metadata({"user_api_key": jwt_hash}) + meta = _get_spend_logs_metadata({"user_api_key": jwt_hash, "user_api_key_hash": jwt_hash}) assert meta["user_api_key"] == jwt_hash +def test_get_spend_logs_metadata_hashed_jwt_shape_without_provenance_is_hashed(): + lookalike = "hashed-jwt-" + "b" * 64 + meta = _get_spend_logs_metadata({"user_api_key": lookalike}) + assert meta["user_api_key"] == hash_token(lookalike) + assert meta["user_api_key"] != lookalike + + def test_get_spend_logs_metadata_none_key_is_none(): meta = _get_spend_logs_metadata({"user_api_key": None}) assert meta["user_api_key"] is None From c7b34da079e962d006e3b109739203703fa152f6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 23:19:17 -0700 Subject: [PATCH 11/12] test(proxy): keep a leaked llm_router out of the next test in the worker The proxy conftest already snapshots master_key and prisma_client around every test, because a value left behind on litellm.proxy.proxy_server poisons the rest of the xdist worker. llm_router has the same problem. The PTU rollup reads the running router out of sys.modules, so a router a sibling test left behind lands in its deployment scan and three test_ptu_flat_cost_rollup tests fail or pass depending on how xdist happens to split the shard. --- tests/test_litellm/proxy/conftest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/conftest.py b/tests/test_litellm/proxy/conftest.py index 61752997f0f..65e12b7d777 100644 --- a/tests/test_litellm/proxy/conftest.py +++ b/tests/test_litellm/proxy/conftest.py @@ -18,6 +18,7 @@ from prisma.errors import ClientNotConnectedError _PROXY_MODULE_GLOBALS_TO_ISOLATE = ( "master_key", "prisma_client", + "llm_router", ) @@ -56,7 +57,10 @@ def pytest_runtest_setup(item): Without this, a leaked value (e.g. master_key set by a sibling test) flips the auth short-circuit in user_api_key_auth and causes unrelated - tests in the same xdist worker to return 401 instead of 200. + tests in the same xdist worker to return 401 instead of 200. A leaked + llm_router does the same to anything that reads the running router out + of sys.modules, such as the PTU rollup's deployment scan, which then + counts a sibling test's deployments as if the proxy owned them. This must be a hook pair, not an autouse fixture: an autouse fixture in the root conftest requests monkeypatch, so monkeypatch's undo stack From a50590f32454ee845775a04087d6cd42d249f37d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 23:41:55 -0700 Subject: [PATCH 12/12] fix(spend-tracking): keep the master key alias readable in spend logs Master-key auth stamps the stable alias litellm_proxy_master_key instead of the raw key, so spend logs carry a readable, non-secret identifier for those rows. The new redaction path only recognized sha256 and hashed-jwt shapes, so it hashed that alias and broke continuity with every master-key row written before this change. The alias joins the recognized non-secret values, still behind the same provenance gate, so a caller who sends the alias string as their own bearer token still gets it hashed. --- .../spend_tracking/spend_tracking_utils.py | 27 ++++---- .../test_spend_tracking_utils.py | 64 +++++++++++++++++-- 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 692200b856c..cba1f9069d3 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -10,6 +10,7 @@ from pydantic import BaseModel import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import ( + LITELLM_PROXY_MASTER_KEY_ALIAS, LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, REDACTED_BY_LITELM_STRING, @@ -67,17 +68,21 @@ def _is_master_key(api_key: str | None, _master_key: str | None) -> bool: _HASHED_JWT_RE = re.compile(r"hashed-jwt-[a-fA-F0-9]{64}") -def _is_prehashed_key_shape(value: str) -> bool: - return is_valid_sha256_hash(value) or _HASHED_JWT_RE.fullmatch(value) is not None +def _is_non_secret_key_value(value: str) -> bool: + return ( + value == LITELLM_PROXY_MASTER_KEY_ALIAS + or is_valid_sha256_hash(value) + or _HASHED_JWT_RE.fullmatch(value) is not None + ) -def _redact_logged_api_key(value: str | None, *, already_hashed: bool = False) -> str | None: +def _redact_logged_api_key(value: str | None, *, already_redacted: bool = False) -> str | None: if not isinstance(value, str) or not value: return None stripped: Final = re.sub(r"(?i)^bearer ", "", value) if not stripped: return None - if already_hashed and _is_prehashed_key_shape(stripped): + if already_redacted and _is_non_secret_key_value(stripped): return stripped return hash_token(stripped) @@ -137,10 +142,10 @@ def _get_spend_logs_metadata( clean_metadata: Final = SpendLogsMetadata(**{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__}) _raw_key: Final = clean_metadata.get("user_api_key") _trusted_hash: Final = metadata.get("user_api_key_hash") - _already_hashed: Final = ( - isinstance(_trusted_hash, str) and _is_prehashed_key_shape(_trusted_hash) and _trusted_hash == _raw_key + _already_redacted: Final = ( + isinstance(_trusted_hash, str) and _is_non_secret_key_value(_trusted_hash) and _trusted_hash == _raw_key ) - clean_metadata["user_api_key"] = _redact_logged_api_key(_raw_key, already_hashed=_already_hashed) + clean_metadata["user_api_key"] = _redact_logged_api_key(_raw_key, already_redacted=_already_redacted) clean_metadata["applied_guardrails"] = applied_guardrails clean_metadata["batch_models"] = batch_models clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata @@ -297,10 +302,10 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs standard_logging_completion_tokens = standard_logging_payload.get("completion_tokens", 0) standard_logging_total_tokens = standard_logging_payload.get("total_tokens", 0) _trusted_hash = metadata.get("user_api_key_hash") - _key_already_hashed = ( - isinstance(_trusted_hash, str) and _is_prehashed_key_shape(_trusted_hash) and _trusted_hash == api_key + _key_already_redacted = ( + isinstance(_trusted_hash, str) and _is_non_secret_key_value(_trusted_hash) and _trusted_hash == api_key ) - api_key = _redact_logged_api_key(api_key, already_hashed=_key_already_hashed) or "" + api_key = _redact_logged_api_key(api_key, already_redacted=_key_already_redacted) or "" if ( standard_logging_payload is not None @@ -308,7 +313,7 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs api_key = ( api_key or _redact_logged_api_key( - standard_logging_payload["metadata"].get("user_api_key_hash"), already_hashed=True + standard_logging_payload["metadata"].get("user_api_key_hash"), already_redacted=True ) or "" ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 92e78e04512..f2dd66ee677 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2625,7 +2625,7 @@ def test_redact_logged_api_key_non_sk_raw_key_is_hashed(): def test_redact_logged_api_key_already_valid_sha256_passes_through_with_flag(): already_hashed = hash_token("sk-some-key") assert len(already_hashed) == 64 - result = _redact_logged_api_key(already_hashed, already_hashed=True) + result = _redact_logged_api_key(already_hashed, already_redacted=True) assert result == already_hashed assert hash_token(already_hashed) != result # no double-hash @@ -2653,7 +2653,7 @@ def test_redact_logged_api_key_long_opaque_token_is_hashed(): def test_redact_logged_api_key_hashed_jwt_passes_through(): jwt_hash = "hashed-jwt-" + "a" * 64 - result = _redact_logged_api_key(jwt_hash, already_hashed=True) + result = _redact_logged_api_key(jwt_hash, already_redacted=True) assert result == jwt_hash @@ -2666,7 +2666,7 @@ def test_redact_logged_api_key_hashed_jwt_shape_without_provenance_is_hashed(): def test_redact_logged_api_key_hashed_jwt_trailing_newline_is_hashed(): trailing = "hashed-jwt-" + "a" * 64 + "\n" - result = _redact_logged_api_key(trailing, already_hashed=True) + result = _redact_logged_api_key(trailing, already_redacted=True) assert result == hash_token(trailing) assert result != trailing @@ -2680,6 +2680,33 @@ def test_redact_logged_api_key_hashed_jwt_short_suffix_is_hashed(): assert result == hash_token(short_jwt) +def test_redact_logged_api_key_master_key_alias_passes_through(): + from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS + + result = _redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS, already_redacted=True) + assert result == LITELLM_PROXY_MASTER_KEY_ALIAS + + +def test_redact_logged_api_key_master_key_alias_without_provenance_is_hashed(): + from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS + + result = _redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS) + assert result == hash_token(LITELLM_PROXY_MASTER_KEY_ALIAS) + assert result != LITELLM_PROXY_MASTER_KEY_ALIAS + + +def test_get_spend_logs_metadata_keeps_master_key_alias_readable(): + from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS + + meta = _get_spend_logs_metadata( + { + "user_api_key": LITELLM_PROXY_MASTER_KEY_ALIAS, + "user_api_key_hash": LITELLM_PROXY_MASTER_KEY_ALIAS, + } + ) + assert meta["user_api_key"] == LITELLM_PROXY_MASTER_KEY_ALIAS + + def test_redact_logged_api_key_bearer_only_returns_none(): # "bearer " with nothing after stripping is equivalent to no key assert _redact_logged_api_key("bearer ") is None @@ -2948,7 +2975,7 @@ class TestSpendLogKeyRedaction: def test_already_hashed_key_unchanged(self): hashed = "bcfe8173f5447f10be0e7fb37aaa8b97829d5c9e0498232152f9d123456789ab" - assert _redact_logged_api_key(hashed, already_hashed=True) == hashed + assert _redact_logged_api_key(hashed, already_redacted=True) == hashed def test_bearer_prefixed_non_sk_key_is_hashed(self): raw = "Bearer some-other-token-format" @@ -2995,6 +3022,33 @@ def test_get_logging_payload_non_sk_raw_key_both_fields_hashed(): assert len(parsed_meta["user_api_key"]) == 64 +def test_get_logging_payload_keeps_master_key_alias_readable(): + from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS + + kwargs = { + "model": "openai/gpt-4.1", + "messages": [{"role": "user", "content": "Hello"}], + "call_type": "acompletion", + "litellm_params": { + "metadata": { + "user_api_key": LITELLM_PROXY_MASTER_KEY_ALIAS, + "user_api_key_hash": LITELLM_PROXY_MASTER_KEY_ALIAS, + "user_api_key_user_id": "test_user", + } + }, + } + payload = get_logging_payload( + kwargs=kwargs, + response_obj=Exception("error"), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert payload["api_key"] == LITELLM_PROXY_MASTER_KEY_ALIAS + parsed_meta = json.loads(payload["metadata"]) + assert parsed_meta["user_api_key"] == LITELLM_PROXY_MASTER_KEY_ALIAS + + @patch("litellm.proxy.proxy_server.master_key", None) @patch("litellm.proxy.proxy_server.general_settings", {}) def test_get_logging_payload_hashes_bearer_prefixed_api_key(): @@ -3503,7 +3557,7 @@ def test_redact_logged_api_key_partial_sha256_is_hashed(): def test_redact_logged_api_key_bearer_already_hashed_passes_through_with_flag(): already_hashed = hash_token("sk-some-key") assert len(already_hashed) == 64 - result = _redact_logged_api_key(f"Bearer {already_hashed}", already_hashed=True) + result = _redact_logged_api_key(f"Bearer {already_hashed}", already_redacted=True) assert result == already_hashed assert hash_token(already_hashed) != result