scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

test: enforce PT012 so a pytest.raises block cannot hide dead assertions (#37748)

Browse source 
* test: enforce PT012 so a pytest.raises block cannot hide dead assertions

`with pytest.raises(...)` stops at the first statement that raises. Anything
sequenced after it inside the block never runs, so an assertion written there is
never checked and the test still reports green.

Two sites were doing exactly that, and both assertions turned out to be wrong
once they started running. tests/llm_translation/test_prompt_factory.py asserted
the bedrock rejection names "requires at least one non-system message", which
holds. tests/proxy_unit_tests/test_proxy_server.py asserted the prisma startup
failure mentions "httpx.ConnectError", which never appears: the failure is an
httpx.ConnectError whose message is "All connection attempts failed", so that
test now asserts the type. Its DATABASE_URL override moves to monkeypatch, since
the old restore sat below the assertion and leaked the invalid URL into every
later DB test the moment the assertion started being able to fail.

The remaining 72 sites are rewritten without changing what they exercise: setup
that cannot raise moves above the block, a nested `patch` moves outside it, and
bodies with real control flow (a stream drain, an if/else on sync_mode, a
retry loop) move into a local closure the block calls.

Fixing PT012 unmasked two B017s, since ruff only reports a blind
pytest.raises(Exception) once the block holds a single statement.
tests/proxy_unit_tests/test_auth_checks.py narrows to the ProxyException
can_key_call_model actually raises. tests/local_testing/test_completion_cost.py
was asserting vertex_ai/medlm-medium has no cost entry, which stopped being true
at some point; that dead first half is gone and the rest of the test, which
checks medlm pricing resolves above zero, now runs instead of being skipped.

* chore(ci): ratchet TQ004 to 768 after the prisma test moved to monkeypatch
This commit is contained in:
ryan-crabbe-berri 2026-08-20 19:36:26 -07:00 • committed by GitHub
parent a4dd1be53b
commit a112ba5f63
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
51 changed files with 311 additions and 193 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
ruff-tests.toml
View file

@ -17,6 +17,9 @@
# B017 `pytest.raises(Exception)` accepts the TypeError a refactor introduced just as
# readily as the rejection under test, so a crash reads as a pass. Narrow to the
# real type, or add `match=` where the code genuinely raises a bare Exception
# PT012 a `pytest.raises` block that runs on past the raising call. Everything after
# that call is dead, so an `assert` sitting there is never checked. Keep the
# block to the call itself and put the assertions below it
#
# No target-version here on purpose: it resolves from requires-python (>=3.10), so
# 3.11-only builtins like BaseExceptionGroup are correctly flagged in a tree that
@ -24,4 +27,4 @@
line-length = 120
lint.select = ["F821", "B011", "B015", "B017", "B018", "PT015", "PLR0133", "PLW0127"]
lint.select = ["F821", "B011", "B015", "B017", "B018", "PT012", "PT015", "PLR0133", "PLW0127"]

2
test-quality-budget.json
View file

@ -9,7 +9,7 @@
"limit": 1078
},
"TQ004": {
"limit": 770
"limit": 768
},
"TQ005": {
"limit": 2832

11
tests/guardrails_tests/test_bedrock_guardrails.py
View file

@ -205,7 +205,7 @@ async def test_bedrock_guardrails_with_streaming():
mock_user_api_key_cache = MagicMock(spec=DualCache)
mock_user_api_key_dict = UserAPIKeyAuth()
with pytest.raises(HTTPException):
async def _stream_through_guardrail():
proxy_logging_obj = ProxyLogging(
user_api_key_cache=mock_user_api_key_cache,
premium_user=True,
@ -240,6 +240,9 @@ async def test_bedrock_guardrails_with_streaming():
async for chunk in response:
print(chunk)
with pytest.raises(HTTPException):
await _stream_through_guardrail()
@pytest.mark.asyncio
async def test_bedrock_guardrails_with_streaming_no_violation():
@ -1502,7 +1505,7 @@ async def test_bedrock_guardrail_disable_exception_on_block_streaming():
mock_post.return_value = mock_bedrock_response
# Should raise exception during streaming processing
with pytest.raises(HTTPException):
async def _drain():
result_generator = (
guardrail_default.async_post_call_streaming_iterator_hook(
user_api_key_dict=mock_user_api_key_dict,
@ -1511,10 +1514,12 @@ async def test_bedrock_guardrail_disable_exception_on_block_streaming():
)
)
# Try to consume the generator - should raise exception
async for chunk in result_generator:
pass
with pytest.raises(HTTPException):
await _drain()
# Test 2: disable_exception_on_block=True. Streaming can't raise up to the
# endpoint handler (SSE headers already flushed), so the block is delivered
# as a synthetic stream with finish_reason=content_filter and the block

5
tests/llm_responses_api_testing/test_openai_responses_api.py
View file

@ -1643,10 +1643,13 @@ async def test_openai_responses_api_token_limit_error():
model="gpt-5-mini", input=oversized_text, stream=True
)
with pytest.raises(litellm.APIError) as exc_info:
async def _drain():
async for event in response:
print(event)
with pytest.raises(litellm.APIError) as exc_info:
await _drain()
assert exc_info.value.status_code == 400
assert "exceeds the context window" in str(exc_info.value)

3
tests/llm_translation/test_prompt_factory.py
View file

@ -1288,7 +1288,8 @@ def test_just_system_message():
model="anthropic.claude-3-sonnet-20240229-v1:0",
llm_provider="bedrock",
)
assert "bedrock requires at least one non-system message" in str(e.value)
assert "bedrock requires at least one non-system message" in str(e.value)
def test_convert_generic_image_chunk_to_openai_image_obj():

41
tests/local_testing/test_aim_guardrails.py
View file

@ -101,26 +101,26 @@ async def test_block_callback(mode: str):
],
}
with pytest.raises(ProxyException, match="Jailbreak detected") as exc_info:
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
return_value=Response(
json={
"analysis_result": {
"analysis_time_ms": 212,
"policy_drill_down": {},
"session_entities": [],
},
"required_action": {
"action_type": "block_action",
"detection_message": "Jailbreak detected",
"policy_name": "blocking policy",
},
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
return_value=Response(
json={
"analysis_result": {
"analysis_time_ms": 212,
"policy_drill_down": {},
"session_entities": [],
},
status_code=200,
request=Request(method="POST", url="http://aim"),
),
):
"required_action": {
"action_type": "block_action",
"detection_message": "Jailbreak detected",
"policy_name": "blocking policy",
},
},
status_code=200,
request=Request(method="POST", url="http://aim"),
),
):
async def _call_guardrail():
if mode == "pre_call":
await aim_guardrail.async_pre_call_hook(
data=data,
@ -135,6 +135,9 @@ async def test_block_callback(mode: str):
call_type="completion",
)
with pytest.raises(ProxyException, match="Jailbreak detected") as exc_info:
await _call_guardrail()
exc = exc_info.value
assert exc.code == "400"
assert exc.type == "invalid_request_error"

8
tests/local_testing/test_completion_cost.py
View file

@ -625,17 +625,9 @@ def test_vertex_ai_completion_cost():
print("calculated_input_cost: {}".format(calculated_input_cost))
@pytest.mark.skip(reason="new test - WIP, working on fixing this")
def test_vertex_ai_medlm_completion_cost():
"""Test for medlm completion cost ."""
with pytest.raises(Exception) as e:
model = "vertex_ai/medlm-medium"
messages = [{"role": "user", "content": "Test MedLM completion cost."}]
predictive_cost = completion_cost(
model=model, messages=messages, custom_llm_provider="vertex_ai"
)
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")

5
tests/local_testing/test_exceptions.py
View file

@ -1417,7 +1417,7 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model):
import litellm
litellm.set_verbose = True
with pytest.raises(Exception) as exc_info:
async def _call_with_bad_role():
if sync_mode:
litellm.completion(
model=model,
@ -1433,6 +1433,9 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model):
sync_stream=sync_mode,
)
with pytest.raises(Exception) as exc_info:
await _call_with_bad_role()
assert exc_info.value.code == "invalid_value"
assert exc_info.value.param is not None
assert exc_info.value.type == "invalid_request_error"

5
tests/local_testing/test_function_calling.py
View file

@ -357,14 +357,13 @@ def test_parallel_function_call_anthropic_error_msg(
if expect_unsupported_params_error:
with pytest.raises(litellm.UnsupportedParamsError) as e:
second_response = litellm.completion(
litellm.completion(
model=model,
messages=messages,
temperature=0.2,
seed=22,
drop_params=True,
) # get a new response from the model where it can see the function response
print("second response\n", second_response)
)
else:
second_response = litellm.completion(
model=model,

3
tests/local_testing/test_mock_request.py
View file

@ -128,13 +128,12 @@ def test_router_mock_request_with_mock_timeout():
],
)
with pytest.raises(litellm.Timeout):
response = router.completion(
router.completion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, I'm a mock request"}],
timeout=3,
mock_timeout=True,
)
print(response)
end_time = time.time()
assert end_time - start_time >= 3, f"Time taken: {end_time - start_time}"

12
tests/local_testing/test_router_budget_limiter.py
View file

@ -161,12 +161,10 @@ async def test_provider_budgets_e2e_test_expect_to_fail():
for _ in range(3):
with pytest.raises(Exception) as exc_info:
response = await router.acompletion(
await router.acompletion(
messages=[{"role": "user", "content": "Hello, how are you?"}],
model="anthropic/claude-sonnet-4-5-20250929",
)
print(response)
print("response.hidden_params", response._hidden_params)
await asyncio.sleep(0.5)
# Verify the error is related to budget exceeded
@ -597,12 +595,10 @@ async def test_deployment_budgets_e2e_test_expect_to_fail():
for _ in range(3):
with pytest.raises(Exception) as exc_info:
response = await router.acompletion(
await router.acompletion(
messages=[{"role": "user", "content": "Hello, how are you?"}],
model="openai/gpt-4o-mini",
)
print(response)
print("response.hidden_params", response._hidden_params)
await asyncio.sleep(0.5)
# Verify the error is related to budget exceeded
@ -651,13 +647,11 @@ async def test_tag_budgets_e2e_test_expect_to_fail():
for _ in range(3):
with pytest.raises(Exception) as exc_info:
response = await router.acompletion(
await router.acompletion(
messages=[{"role": "user", "content": "Hello, how are you?"}],
model="openai/gpt-4o-mini",
metadata={"tags": [TAG_NAME]},
)
print(response)
print("response.hidden_params", response._hidden_params)
await asyncio.sleep(0.5)
# Verify the error is related to budget exceeded

5
tests/local_testing/test_router_fallbacks.py
View file

@ -1416,7 +1416,7 @@ async def test_router_fallbacks_default_and_model_specific_fallbacks(sync_mode):
default_fallbacks=["bad-model"],
)
with pytest.raises(Exception) as exc_info:
async def _call_bad_model():
if sync_mode:
resp = router.completion(
model="bad-model",
@ -1429,6 +1429,9 @@ async def test_router_fallbacks_default_and_model_specific_fallbacks(sync_mode):
model="bad-model",
messages=[{"role": "user", "content": "Hey, how's it going?"}],
)
with pytest.raises(Exception) as exc_info:
await _call_bad_model()
assert isinstance(
exc_info.value, litellm.AuthenticationError
), f"Expected AuthenticationError, but got {type(exc_info.value).__name__}"

5
tests/local_testing/test_router_max_parallel_requests.py
View file

@ -205,9 +205,12 @@ async def test_max_parallel_requests_tpm_rate_limiting_base_case():
num_retries=0,
)
with pytest.raises(litellm.RateLimitError):
async def _exceed_limit():
for _ in range(2):
await router.acompletion(
model="gpt-4o-2024-08-06",
messages=_messages,
)
with pytest.raises(litellm.RateLimitError):
await _exceed_limit()

7
tests/local_testing/test_streaming.py
View file

@ -2926,11 +2926,14 @@ def test_unit_test_custom_stream_wrapper_repeating_chunk(
print(f"expected_chunk_fail: {expected_chunk_fail}")
if (loop_amount > litellm.REPEATED_STREAMING_CHUNK_LIMIT) and expected_chunk_fail:
def _drain():
for chunk in response:
continue
with pytest.raises(
(litellm.InternalServerError, litellm.exceptions.MidStreamFallbackError)
):
for chunk in response:
continue
_drain()
else:
for chunk in response:
continue

5
tests/proxy_admin_ui_tests/test_access_group_team_sync.py
View file

@ -170,12 +170,15 @@ async def test_a_failed_mirror_takes_the_new_team_row_with_it():
async with _clean_db() as db:
await _seed(db, {GROUPS[0]: [], GROUPS[1]: [OTHER_TEAM]})
with pytest.raises(RuntimeError):
async def _blow_up_after_reconcile():
async with db.tx() as tx:
await tx.litellm_teamtable.create(data={"team_id": TEAM, "access_group_ids": [GROUPS[0]]})
await reconcile_team_access_group_membership(tx, TEAM)
raise RuntimeError("the cache handoff blew up")
with pytest.raises(RuntimeError):
await _blow_up_after_reconcile()
assert await _read(db) == {GROUPS[0]: [], GROUPS[1]: [OTHER_TEAM]}
assert await db.litellm_teamtable.find_unique(where={"team_id": TEAM}) is None

6
tests/proxy_unit_tests/test_auth_checks.py
View file

@ -242,8 +242,8 @@ async def test_can_team_call_model(model, expect_to_work):
)
@pytest.mark.asyncio
async def test_can_key_call_model_wildcard_access(key_models, model, expect_to_work):
from litellm.proxy._types import ProxyException
from litellm.proxy.auth.auth_checks import can_key_call_model
from fastapi import HTTPException
llm_model_list = [
{
@ -294,7 +294,7 @@ async def test_can_key_call_model_wildcard_access(key_models, model, expect_to_w
llm_router=router,
)
else:
with pytest.raises(Exception) as e:
with pytest.raises(ProxyException):
await can_key_call_model(
model=model,
llm_model_list=llm_model_list,
@ -302,8 +302,6 @@ async def test_can_key_call_model_wildcard_access(key_models, model, expect_to_w
llm_router=router,
)
print(e)
@pytest.mark.parametrize(
"key_models, model, expect_to_work",

3
tests/proxy_unit_tests/test_jwt.py
View file

@ -1047,8 +1047,7 @@ async def test_allow_access_by_email(
else:
# Expect the call to fail
with pytest.raises(ProxyException):
resp = await user_api_key_auth(request=request, api_key=bearer_token)
print(resp)
await user_api_key_auth(request=request, api_key=bearer_token)
def test_get_public_key_from_jwk_url():

18
tests/proxy_unit_tests/test_proxy_server.py
View file

@ -2412,12 +2412,14 @@ async def test_proxy_server_prisma_setup():
@pytest.mark.asyncio
async def test_proxy_server_prisma_setup_invalid_db():
async def test_proxy_server_prisma_setup_invalid_db(monkeypatch):
"""
PROD TEST: Test that proxy server startup fails when it's unable to connect to the database
Think 2-3 times before editing / deleting this test, it's important for PROD
"""
import httpx
from litellm.proxy.proxy_server import ProxyStartupEvent
from litellm.proxy.utils import ProxyLogging
from litellm.caching import DualCache
@ -2425,24 +2427,14 @@ async def test_proxy_server_prisma_setup_invalid_db():
user_api_key_cache = DualCache()
invalid_db_url = "postgresql://invalid:invalid@localhost:5432/nonexistent"
_old_db_url = os.getenv("DATABASE_URL")
os.environ["DATABASE_URL"] = invalid_db_url
monkeypatch.setenv("DATABASE_URL", invalid_db_url)
with pytest.raises(Exception) as exc_info:
with pytest.raises(httpx.ConnectError):
await ProxyStartupEvent._setup_prisma_client(
database_url=invalid_db_url,
proxy_logging_obj=ProxyLogging(user_api_key_cache=user_api_key_cache),
user_api_key_cache=user_api_key_cache,
)
print("GOT EXCEPTION=", exc_info)
assert "httpx.ConnectError" in str(exc_info.value)
# # Verify the error message indicates a database connection issue
# assert any(x in str(exc_info.value).lower() for x in ["database", "connection", "authentication"])
if _old_db_url:
os.environ["DATABASE_URL"] = _old_db_url
@pytest.mark.asyncio

14
tests/router_unit_tests/test_router_helper_utils.py
View file

@ -423,10 +423,11 @@ def test_get_timeout(model_list):
def test_handle_mock_testing_fallbacks(model_list, fallback_kwarg, expected_error):
"""Test if the '_handle_mock_testing_fallbacks' function is working correctly"""
router = Router(model_list=model_list)
data = {
fallback_kwarg: True,
}
with pytest.raises(expected_error):
data = {
fallback_kwarg: True,
}
router._handle_mock_testing_fallbacks(
kwargs=data,
)
@ -435,10 +436,11 @@ def test_handle_mock_testing_fallbacks(model_list, fallback_kwarg, expected_erro
def test_handle_mock_testing_rate_limit_error(model_list):
"""Test if the '_handle_mock_testing_rate_limit_error' function is working correctly"""
router = Router(model_list=model_list)
data = {
"mock_testing_rate_limit_error": True,
}
with pytest.raises(litellm.RateLimitError):
data = {
"mock_testing_rate_limit_error": True,
}
router._handle_mock_testing_rate_limit_error(
kwargs=data,
)

7
tests/test_litellm/a2a_protocol/test_a2a_exception_mapping_utils.py
View file

@ -171,9 +171,12 @@ async def test_stream_with_retry_raises_after_localhost_retries_exhausted():
api_base="https://agent.example",
agent_name="test-agent",
)
async def _drain():
async for _chunk in stream:
pytest.fail("expected retry exhaustion to raise before yielding")
with pytest.raises(
RuntimeError,
match="no response received after retry attempts",
):
async for _chunk in stream:
pytest.fail("expected retry exhaustion to raise before yielding")
await _drain()

9
tests/test_litellm/caching/test_redis_semantic_cache.py
View file

@ -310,11 +310,12 @@ def test_redis_semantic_cache_reraises_unexpected_isolated_index_error(monkeypat
monkeypatch.setenv("REDIS_PORT", "6379")
monkeypatch.setenv("REDIS_PASSWORD", "test_password")
cache = RedisSemanticCache(
similarity_threshold=0.8,
index_name="existing_index",
)
with pytest.raises(ValueError, match="connection failed"):
cache = RedisSemanticCache(
similarity_threshold=0.8,
index_name="existing_index",
)
_ = cache.llmcache

7
tests/test_litellm/experimental_mcp_client/test_mcp_client.py
View file

@ -75,11 +75,10 @@ class TestMCPClient:
# Test missing stdio_config
client = MCPClient(transport_type=MCPTransport.stdio)
async def _noop(session):
return None
with pytest.raises(ValueError, match="stdio_config is required for stdio transport"):
async def _noop(session):
return None
await client.run_with_session(_noop)
@pytest.mark.asyncio

31
tests/test_litellm/integrations/bitbucket/test_bitbucket_integration.py
View file

@ -88,39 +88,44 @@ def test_bitbucket_prompt_manager_error_handling(mock_client_class):
"access_token": "test-token",
}
manager = BitBucketPromptManager(config, prompt_id="test_prompt")
with pytest.raises(
Exception, match="Failed to load prompt 'test_prompt' from BitBucket"
):
manager = BitBucketPromptManager(config, prompt_id="test_prompt")
_ = manager.prompt_manager # This triggers the error
_ = manager.prompt_manager
def test_bitbucket_prompt_manager_config_validation():
"""Test BitBucketPromptManager configuration validation."""
# Test missing required fields - validation happens when prompt_manager is accessed
with pytest.raises(
ValueError, match="workspace, repository, and access_token are required"
):
manager = BitBucketPromptManager({})
_ = manager.prompt_manager # This triggers validation
manager = BitBucketPromptManager({})
with pytest.raises(
ValueError, match="workspace, repository, and access_token are required"
):
manager = BitBucketPromptManager({"workspace": "test"})
_ = manager.prompt_manager # This triggers validation
_ = manager.prompt_manager
manager = BitBucketPromptManager({"workspace": "test"})
with pytest.raises(
ValueError, match="workspace, repository, and access_token are required"
):
manager = BitBucketPromptManager({"repository": "test"})
_ = manager.prompt_manager # This triggers validation
_ = manager.prompt_manager
manager = BitBucketPromptManager({"repository": "test"})
with pytest.raises(
ValueError, match="workspace, repository, and access_token are required"
):
manager = BitBucketPromptManager({"access_token": "test"})
_ = manager.prompt_manager # This triggers validation
_ = manager.prompt_manager
manager = BitBucketPromptManager({"access_token": "test"})
with pytest.raises(
ValueError, match="workspace, repository, and access_token are required"
):
_ = manager.prompt_manager
@patch("litellm.integrations.bitbucket.bitbucket_prompt_manager.BitBucketClient")

10
tests/test_litellm/litellm_core_utils/test_streaming_handler.py
View file

@ -2143,10 +2143,13 @@ def test_raise_on_model_repetition(
chunks = _build_chunks(chunks_pattern, len(chunks_pattern))
if should_raise:
with pytest.raises(litellm.InternalServerError) as exc_info:
def _feed():
for chunk in chunks:
wrapper.chunks.append(chunk)
wrapper.raise_on_model_repetition()
with pytest.raises(litellm.InternalServerError) as exc_info:
_feed()
assert "repeating the same chunk" in str(exc_info.value)
else:
for chunk in chunks:
@ -3616,10 +3619,13 @@ async def test_transport_read_error_before_finish_reason_raises(logging_obj: Log
)
received = []
with pytest.raises(MidStreamFallbackError):
async def _drain():
async for chunk in response:
received.append(chunk)
with pytest.raises(MidStreamFallbackError):
await _drain()
fabricated_finish_reasons = [
chunk.choices[0].finish_reason
for chunk in received

6
tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -1986,10 +1986,11 @@ def test_effort_validation():
)
assert result["output_config"]["effort"] == effort
optional_params = {"output_config": {"effort": "invalid"}}
with pytest.raises(
litellm.exceptions.BadRequestError, match="Invalid effort value"
):
optional_params = {"output_config": {"effort": "invalid"}}
config.transform_request(
model="claude-opus-4-5-20251101",
messages=messages,
@ -2043,11 +2044,12 @@ def test_max_effort_rejected_for_opus_45():
messages = [{"role": "user", "content": "Test"}]
optional_params = {"output_config": {"effort": "max"}}
with pytest.raises(
litellm.exceptions.BadRequestError,
match="effort='max' is not supported by this model",
):
optional_params = {"output_config": {"effort": "max"}}
config.transform_request(
model="claude-opus-4-5-20251101",
messages=messages,

7
tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py
View file

@ -35,11 +35,10 @@ class TestBytezChatConfig:
assert result["user-agent"] == f"litellm/{version}"
def test_missing_api_key(self):
config = BytezChatConfig()
headers = {}
with pytest.raises(Exception) as excinfo:
config = BytezChatConfig()
headers = {}
config.validate_environment(
headers=headers,
model=TEST_MODEL,

25
tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py
View file

@ -127,10 +127,13 @@ async def test_client_payload_error_mid_stream_raises_read_error():
stream = AiohttpResponseStream(mock_response) # type: ignore
received_chunks = []
with pytest.raises(httpx.ReadError):
async def _drain():
async for chunk in stream:
received_chunks.append(chunk)
with pytest.raises(httpx.ReadError):
await _drain()
assert received_chunks == [b"chunk1"]
assert mock_response.closed is True
@ -151,10 +154,13 @@ async def test_client_payload_error_before_first_chunk_raises_read_error():
stream = AiohttpResponseStream(mock_response) # type: ignore
received_chunks = []
with pytest.raises(httpx.ReadError):
async def _drain():
async for chunk in stream:
received_chunks.append(chunk)
with pytest.raises(httpx.ReadError):
await _drain()
assert received_chunks == []
assert mock_response.closed is True
@ -171,10 +177,13 @@ async def test_connection_closed_runtime_error_raises_read_error():
stream = AiohttpResponseStream(mock_response) # type: ignore
received_chunks = []
with pytest.raises(httpx.ReadError):
async def _drain():
async for chunk in stream:
received_chunks.append(chunk)
with pytest.raises(httpx.ReadError):
await _drain()
assert received_chunks == [b"data1"]
assert mock_response.closed is True
@ -209,10 +218,13 @@ async def test_transfer_encoding_error_raises_read_error():
stream = AiohttpResponseStream(mock_response) # type: ignore
received_chunks = []
with pytest.raises(httpx.ReadError):
async def _drain():
async for chunk in stream:
received_chunks.append(chunk)
with pytest.raises(httpx.ReadError):
await _drain()
assert received_chunks == [b"data1"]
assert mock_response.closed is True
@ -254,10 +266,13 @@ async def test_timeout_exception_gets_mapped():
received_chunks = []
# This should raise httpx.TimeoutException (mapped from aiohttp.ServerTimeoutError)
with pytest.raises(httpx.TimeoutException):
async def _drain():
async for chunk in stream:
received_chunks.append(chunk)
with pytest.raises(httpx.TimeoutException):
await _drain()
# Should have received the first chunk before the error
assert received_chunks == [b"chunk1"]

14
tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py
View file

@ -287,10 +287,11 @@ class TestHTTPHandlerErrorPaths:
"send",
side_effect=_make_httpx_status_error(url="https://api.test.com?key=SECRET"),
):
kwargs = {"url": "https://api.test.com?key=SECRET"}
if method != "delete":
kwargs["data"] = {"test": 1}
with pytest.raises(MaskedHTTPStatusError) as exc_info:
kwargs = {"url": "https://api.test.com?key=SECRET"}
if method != "delete":
kwargs["data"] = {"test": 1}
getattr(sync_handler, method)(**kwargs)
assert "SECRET" not in str(exc_info.value.request.url)
@ -304,10 +305,11 @@ class TestHTTPHandlerErrorPaths:
new_callable=AsyncMock,
side_effect=_make_httpx_status_error(url="https://api.test.com?key=SECRET"),
):
kwargs = {"url": "https://api.test.com?key=SECRET"}
if method != "delete":
kwargs["data"] = {"test": 1}
with pytest.raises(MaskedHTTPStatusError) as exc_info:
kwargs = {"url": "https://api.test.com?key=SECRET"}
if method != "delete":
kwargs["data"] = {"test": 1}
await getattr(async_handler, method)(**kwargs)
assert "SECRET" not in str(exc_info.value.request.url)

6
tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py
View file

@ -95,10 +95,10 @@ class TestOCIChatConfig:
modified_params = params.copy()
del modified_params[key]
with pytest.raises(Exception) as excinfo:
config = OCIChatConfig()
headers = {}
config = OCIChatConfig()
headers = {}
with pytest.raises(Exception) as excinfo:
config.validate_environment(
headers=headers,
model=TEST_MODEL,

10
tests/test_litellm/llms/openai/test_openai_common_utils.py
View file

@ -375,20 +375,26 @@ async def test_async_streaming_output_limit_400_maps_to_length_truncated_stream(
@pytest.mark.parametrize("provider", ["openai", "azure"])
@pytest.mark.parametrize("stream", [False, True])
def test_sync_genuine_bad_request_still_raises(provider, stream):
with pytest.raises(litellm.BadRequestError):
def _call_and_drain():
result = litellm.completion(
**_completion_kwargs(provider, _sync_client_raising(provider, GENUINE_400_MESSAGE), stream=stream)
)
list(result)
with pytest.raises(litellm.BadRequestError):
_call_and_drain()
@pytest.mark.parametrize("provider", ["openai", "azure"])
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.asyncio
async def test_async_genuine_bad_request_still_raises(provider, stream):
with pytest.raises(litellm.BadRequestError):
async def _call_and_drain():
result = await litellm.acompletion(
**_completion_kwargs(provider, _async_client_raising(provider, GENUINE_400_MESSAGE), stream=stream)
)
async for _ in result:
pass
with pytest.raises(litellm.BadRequestError):
await _call_and_drain()

5
tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
View file

@ -5246,11 +5246,14 @@ def test_mid_stream_429_error_raises_during_iteration():
# Iterate the stream: first chunks should succeed, then 429 error should be raised
results = []
with pytest.raises(VertexAIError) as exc_info:
def _drain():
for chunk in streaming_obj:
if chunk is not None:
results.append(chunk)
with pytest.raises(VertexAIError) as exc_info:
_drain()
# Verify: received normal chunks before the error
assert (
len(results) >= 1

7
tests/test_litellm/llms/volcengine/test_volcengine_embedding.py
View file

@ -198,10 +198,11 @@ def test_volcengine_embedding_error_scenarios():
mock_embedding.side_effect = ValueError("Unsupported encoding_format")
# Test that errors are properly raised
test_params = {
k: v for k, v in scenario.items() if k != "expected_error_pattern"
}
with pytest.raises(Exception) as exc_info:
test_params = {
k: v for k, v in scenario.items() if k != "expected_error_pattern"
}
litellm.embedding(input=["test"], **test_params)
# Verify error message contains expected pattern

5
tests/test_litellm/passthrough/test_async_streaming_error_propagation.py
View file

@ -65,7 +65,7 @@ async def test_async_streaming_429_raises():
return mock_response
chunks = []
with pytest.raises(httpx.HTTPStatusError) as exc_info:
async def _drain():
async for chunk in _async_streaming(
response=response_coro(),
litellm_logging_obj=_make_mock_logging_obj(),
@ -73,6 +73,9 @@ async def test_async_streaming_429_raises():
):
chunks.append(chunk)
with pytest.raises(httpx.HTTPStatusError) as exc_info:
await _drain()
assert exc_info.value.response.status_code == 429
assert len(chunks) == 0

5
tests/test_litellm/passthrough/test_passthrough_main.py
View file

@ -721,10 +721,13 @@ async def test_allm_passthrough_route_429_streaming_raises():
# result is an async generator — consuming it must raise, not silently yield error bytes
chunks = []
with pytest.raises(httpx.HTTPStatusError) as exc_info:
async def _drain():
async for chunk in result: # type: ignore[union-attr]
chunks.append(chunk)
with pytest.raises(httpx.HTTPStatusError) as exc_info:
await _drain()
assert exc_info.value.response.status_code == 429
assert len(chunks) == 0, "No chunks should be yielded before the 429 raises"

5
tests/test_litellm/passthrough/test_streaming_interrupt_spend_tracking.py
View file

@ -164,7 +164,7 @@ async def test_async_streaming_flushes_on_upstream_exception_with_partial_data()
provider_config = MagicMock()
received = []
with pytest.raises(httpx.ReadError):
async def _drain():
async for chunk in _async_streaming(
response=response_coro(),
litellm_logging_obj=mock_logging_obj,
@ -172,6 +172,9 @@ async def test_async_streaming_flushes_on_upstream_exception_with_partial_data()
):
received.append(chunk)
with pytest.raises(httpx.ReadError):
await _drain()
assert received == partial_chunks
await asyncio.sleep(0)

11
tests/test_litellm/proxy/auth/test_auth_exception_handler.py
View file

@ -338,12 +338,13 @@ async def test_handle_authentication_error_budget_exceeded():
mock_api_key = "test-key"
# Test with budget exceeded error
with pytest.raises(ProxyException) as exc_info:
from litellm.exceptions import BudgetExceededError
from litellm.exceptions import BudgetExceededError
budget_error = BudgetExceededError(
message="Budget exceeded", current_cost=100, max_budget=100
)
budget_error = BudgetExceededError(
message="Budget exceeded", current_cost=100, max_budget=100
)
with pytest.raises(ProxyException) as exc_info:
await handler._handle_authentication_error(
budget_error,
mock_request,

9
tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py
View file

@ -487,17 +487,18 @@ async def test_openai_moderation_guardrail_streaming_harmful_content():
# Should raise HTTPException when processing streaming harmful content
from fastapi import HTTPException
with pytest.raises(HTTPException) as exc_info:
async def _drain():
result_chunks = []
async for (
chunk
) in unified_guardrail.async_post_call_streaming_iterator_hook(
async for chunk in unified_guardrail.async_post_call_streaming_iterator_hook(
user_api_key_dict=user_api_key_dict,
response=mock_stream(),
request_data=request_data,
):
result_chunks.append(chunk)
with pytest.raises(HTTPException) as exc_info:
await _drain()
assert exc_info.value.status_code == 400
assert "Violated OpenAI moderation policy" in str(exc_info.value.detail)

41
tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cato_networks.py
View file

@ -93,26 +93,26 @@ async def test_block_callback(mode: str):
],
}
with pytest.raises(HTTPException, match="Jailbreak detected"):
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
return_value=Response(
json={
"analysis_result": {
"analysis_time_ms": 212,
"policy_drill_down": {},
"session_entities": [],
},
"required_action": {
"action_type": "block_action",
"detection_message": "Jailbreak detected",
"policy_name": "blocking policy",
},
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
return_value=Response(
json={
"analysis_result": {
"analysis_time_ms": 212,
"policy_drill_down": {},
"session_entities": [],
},
status_code=200,
request=Request(method="POST", url="http://cato"),
),
):
"required_action": {
"action_type": "block_action",
"detection_message": "Jailbreak detected",
"policy_name": "blocking policy",
},
},
status_code=200,
request=Request(method="POST", url="http://cato"),
),
):
async def _call_guardrail():
if mode == "pre_call":
await cato_guardrail.async_pre_call_hook(
data=data,
@ -127,6 +127,9 @@ async def test_block_callback(mode: str):
call_type="completion",
)
with pytest.raises(HTTPException, match="Jailbreak detected"):
await _call_guardrail()
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["pre_call", "during_call"])

15
tests/test_litellm/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py
View file

@ -2441,7 +2441,7 @@ class TestStreamingIteratorHook:
),
):
chunks = []
with pytest.raises(HTTPException) as exc_info:
async def _drain():
async for chunk in guardrail.async_post_call_streaming_iterator_hook(
user_api_key_dict=UserAPIKeyAuth(
api_key="test", user_id="user-123"
@ -2451,6 +2451,9 @@ class TestStreamingIteratorHook:
):
chunks.append(chunk)
with pytest.raises(HTTPException) as exc_info:
await _drain()
assert exc_info.value.status_code == 400
assert len(chunks) == 0 # No chunks yielded before the block
@ -2477,7 +2480,7 @@ class TestStreamingIteratorHook:
"litellm.main.stream_chunk_builder", return_value=assembled_response
):
chunks = []
with pytest.raises(HTTPException) as exc_info:
async def _drain():
async for chunk in guardrail.async_post_call_streaming_iterator_hook(
user_api_key_dict=UserAPIKeyAuth(api_key="test"), # no user_id
response=fake_response_stream(),
@ -2485,6 +2488,9 @@ class TestStreamingIteratorHook:
):
chunks.append(chunk)
with pytest.raises(HTTPException) as exc_info:
await _drain()
assert exc_info.value.status_code == 400
assert len(chunks) == 0
@ -2625,7 +2631,7 @@ class TestStreamingIteratorHook:
),
):
chunks = []
with pytest.raises(HTTPException) as exc_info:
async def _drain():
async for chunk in guardrail.async_post_call_streaming_iterator_hook(
user_api_key_dict=UserAPIKeyAuth(
api_key="test", user_id="user-123"
@ -2635,6 +2641,9 @@ class TestStreamingIteratorHook:
):
chunks.append(chunk)
with pytest.raises(HTTPException) as exc_info:
await _drain()
assert exc_info.value.status_code == 400
assert len(chunks) == 0

5
tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py
View file

@ -5902,7 +5902,7 @@ class TestPanwAirsBlockedErrorDetailPassthrough:
with patch.object(
base_handler, "_call_panw_api", return_value=copy.deepcopy(self._FULL_BLOCK_RESPONSE)
):
with pytest.raises(HTTPException) as exc_info:
async def _call_hook():
if is_response:
await base_handler.async_post_call_success_hook(
data=safe_prompt_data,
@ -5917,6 +5917,9 @@ class TestPanwAirsBlockedErrorDetailPassthrough:
call_type="completion",
)
with pytest.raises(HTTPException) as exc_info:
await _call_hook()
error = exc_info.value.detail["error"]
for field, value in self._FULL_BLOCK_RESPONSE.items():
if field == "category":

6
tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py
View file

@ -472,9 +472,9 @@ async def test_file_sanitization_block():
async def mock_get(*args, **kwargs):
return mock_poll_response
with pytest.raises(HTTPException) as excinfo:
with patch.object(guardrail.async_handler, "post", side_effect=mock_post):
with patch.object(guardrail.async_handler, "get", side_effect=mock_get):
with patch.object(guardrail.async_handler, "post", side_effect=mock_post):
with patch.object(guardrail.async_handler, "get", side_effect=mock_get):
with pytest.raises(HTTPException) as excinfo:
await guardrail.apply_guardrail(
inputs=inputs,
request_data=request_data,

3
tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
View file

@ -11037,8 +11037,7 @@ class TestKeyAliasSkipValidationOnUnchanged:
assert new_alias != existing_alias
with pytest.raises(ProxyException):
if new_alias != existing_alias:
_validate_key_alias_format(new_alias)
_validate_key_alias_format(new_alias)
@pytest.mark.asyncio
async def test_update_key_changed_to_valid_alias_passes(

5
tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py
View file

@ -573,12 +573,15 @@ async def test_http_validator_service_outage_fails_closed(monkeypatch, kind, exi
monkeypatch.setenv("TEAM_METADATA_VALIDATION_SERVICE_URL", _closed_port_url())
with _configured(impls.validate_via_http):
with pytest.raises(ProxyException) as exc_info:
async def _drive():
if kind == "create":
await _drive_create(metadata=request_payload)
else:
await _drive_update(kind, existing_metadata, request_payload)
with pytest.raises(ProxyException) as exc_info:
await _drive()
assert str(exc_info.value.code) == "503"
assert DEFAULT_TEAM_METADATA_VALIDATION_UNAVAILABLE_MESSAGE in str(exc_info.value.message)

15
tests/test_litellm/proxy/test_budget_reservation.py
View file

@ -2412,10 +2412,13 @@ async def test_streaming_cancel_before_any_chunk_reconciles_to_input_cost(
generator, streaming_logging_obj = _drive_streaming_cancel(valid_token, cancel_before_chunk)
received = []
with pytest.raises(asyncio.CancelledError):
async def _drain():
async for chunk in generator:
received.append(chunk)
with pytest.raises(asyncio.CancelledError):
await _drain()
assert received == []
# no chunk delivered, but the provider already received the input, so the
# reservation is reconciled to the input cost (0.5), not refunded to zero
@ -2444,10 +2447,13 @@ async def test_streaming_cancel_after_chunk_keeps_reservation(
generator, streaming_logging_obj = _drive_streaming_cancel(valid_token, cancel_after_chunk)
received = []
with pytest.raises(asyncio.CancelledError):
async def _drain():
async for chunk in generator:
received.append(chunk)
with pytest.raises(asyncio.CancelledError):
await _drain()
assert received == ["data: chunk\n\n"]
# a consumed stream must NOT be refunded
assert counter_cache.in_memory_cache.get_cache(
@ -2508,10 +2514,13 @@ async def test_streaming_cancel_in_slow_path_before_yield_refunds(spend_counter_
received = []
# include_cost_in_streaming_usage forces fast_path off, so the hook above runs
with patch.object(litellm, "include_cost_in_streaming_usage", True, create=True):
with pytest.raises(asyncio.CancelledError):
async def _drain():
async for chunk in generator:
received.append(chunk)
with pytest.raises(asyncio.CancelledError):
await _drain()
assert received == []
# cancellation happened before any chunk reached the client, but the
# provider already received the input -> reconcile to the input cost (0.5)

10
tests/test_litellm/proxy/test_proxy_logging_hook_detection.py
View file

@ -291,7 +291,7 @@ async def test_post_call_stream_guardrail_blocks_anthropic_messages_stream(monke
yield chunk
delivered = []
with pytest.raises(HTTPException) as exc_info:
async def _drain():
async for chunk in proxy_logging.async_post_call_streaming_iterator_hook(
response=fake_stream(),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/messages"),
@ -299,6 +299,9 @@ async def test_post_call_stream_guardrail_blocks_anthropic_messages_stream(monke
):
delivered.append(chunk)
with pytest.raises(HTTPException) as exc_info:
await _drain()
detail = exc_info.value.detail
assert detail["guardrail_name"] == "output-filter"
assert detail["keyword"] == "zebra"
@ -411,7 +414,7 @@ async def test_post_call_stream_guardrail_reroutes_inherited_apply_guardrail(mon
yield chunk
delivered = []
with pytest.raises(HTTPException) as exc_info:
async def _drain():
async for chunk in proxy_logging.async_post_call_streaming_iterator_hook(
response=fake_stream(),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/messages"),
@ -419,6 +422,9 @@ async def test_post_call_stream_guardrail_reroutes_inherited_apply_guardrail(mon
):
delivered.append(chunk)
with pytest.raises(HTTPException) as exc_info:
await _drain()
assert exc_info.value.detail["keyword"] == "zebra"
assert delivered == []

5
tests/test_litellm/proxy/test_route_llm_request.py
View file

@ -169,7 +169,7 @@ async def test_route_request_proxy_admin_can_call_all_team_scoped_deployments_wi
)
)
with pytest.raises(litellm.BadRequestError, match="multiple teams"):
async def _route_and_await():
ambiguous_call = await route_request(
data=data,
llm_router=router,
@ -179,6 +179,9 @@ async def test_route_request_proxy_admin_can_call_all_team_scoped_deployments_wi
)
await ambiguous_call
with pytest.raises(litellm.BadRequestError, match="multiple teams"):
await _route_and_await()
router.add_deployment(
Deployment(
model_name="team-azure",

10
tests/test_litellm/repositories/test_unit_of_work.py
View file

@ -59,11 +59,14 @@ async def test_updates_across_tables_share_one_batch_and_commit_once():
async def test_raising_inside_block_skips_commit():
batch = FakeBatch()
with pytest.raises(RuntimeError, match="boom"):
async def _blow_up_mid_transaction():
async with spend_reset_unit_of_work(lambda: batch) as uow:
uow.keys.queue_spend_reset(token="tok-1", budget_reset_at=None)
raise RuntimeError("boom")
with pytest.raises(RuntimeError, match="boom"):
await _blow_up_mid_transaction()
assert batch.commit_count == 0
@ -119,9 +122,12 @@ async def test_budget_cascade_raising_inside_block_skips_commit():
the tier is still due on the next tick."""
batch = FakeBatch()
with pytest.raises(RuntimeError, match="boom"):
async def _blow_up_mid_transaction():
async with budget_cascade_unit_of_work(lambda: batch) as uow:
uow.team_memberships.queue_spend_zero(where={"budget_id": {"in": ["budget-1"]}})
raise RuntimeError("boom")
with pytest.raises(RuntimeError, match="boom"):
await _blow_up_mid_transaction()
assert batch.commit_count == 0

5
tests/test_litellm/responses/test_streaming_iterator_error_events.py
View file

@ -208,9 +208,12 @@ async def test_async_iterator_error_after_first_chunk_carries_generated_content(
)
chunks = []
with pytest.raises(MidStreamFallbackError) as exc_info:
async def _drain():
async for chunk in iterator:
chunks.append(chunk)
with pytest.raises(MidStreamFallbackError) as exc_info:
await _drain()
assert len(chunks) == 2
assert exc_info.value.status_code == 500
assert exc_info.value.is_pre_first_chunk is False

8
tests/test_litellm/secret_managers/test_custom_secret_manager.py
View file

@ -243,17 +243,17 @@ def test_minimal_custom_secret_manager():
assert value == "sync-TEST_KEY-value"
# Write should raise NotImplementedError
with pytest.raises(NotImplementedError) as exc_info:
import asyncio
import asyncio
with pytest.raises(NotImplementedError) as exc_info:
asyncio.run(secret_manager.async_write_secret("KEY", "value"))
assert "Write operations are not implemented" in str(exc_info.value)
# Delete should raise NotImplementedError
with pytest.raises(NotImplementedError) as exc_info:
import asyncio
import asyncio
with pytest.raises(NotImplementedError) as exc_info:
asyncio.run(secret_manager.async_delete_secret("KEY"))
assert "Delete operations are not implemented" in str(exc_info.value)

15
tests/test_litellm/test_router.py
View file

@ -1999,10 +1999,13 @@ async def test_acompletion_streaming_iterator():
# Collect streamed chunks — the first chunk succeeds, then the error re-raises
collected_chunks = []
with pytest.raises(MidStreamFallbackError):
async def _drain():
async for chunk in result:
collected_chunks.append(chunk)
with pytest.raises(MidStreamFallbackError):
await _drain()
assert len(collected_chunks) == 1, "one chunk yielded before the error"
print("✓ MidStreamFallbackError re-raised correctly when content was already generated")
@ -5557,10 +5560,13 @@ async def test_acompletion_streaming_iterator_does_not_log_success_on_terminal_f
initial_kwargs=dict(initial_kwargs),
)
collected = []
with pytest.raises(MidStreamFallbackError):
async def _drain():
async for chunk in result:
collected.append(chunk)
with pytest.raises(MidStreamFallbackError):
await _drain()
assert len(collected) == 1
logging_obj.dispatch_success_handlers.assert_not_called()
@ -5580,10 +5586,13 @@ async def test_acompletion_streaming_iterator_does_not_log_success_on_terminal_f
initial_kwargs=dict(initial_kwargs),
)
collected = []
with pytest.raises(MidStreamFallbackError):
async def _drain():
async for chunk in result:
collected.append(chunk)
with pytest.raises(MidStreamFallbackError):
await _drain()
assert len(collected) == 1, "only the partial chunk before the error"
mock_fallback.assert_not_called()
logging_obj.dispatch_success_handlers.assert_not_called()

19
tests/test_ratelimit.py
View file

@ -149,19 +149,26 @@ def test_async_rate_limit(
router: Router = router_factory(rpm, tpm, routing_strategy)
print(f"router: {router.model_list}")
with pytest.raises(expected_exception) as excinfo: # asserts correct type raised
if sync_mode:
results = sync_call(router, list_of_messages)
else:
results = asyncio.run(async_call(router, list_of_messages))
received = []
def _send_and_check():
results = (
sync_call(router, list_of_messages)
if sync_mode
else asyncio.run(async_call(router, list_of_messages))
)
received.extend(results)
print(results)
if len([i for i in results if i is not None]) != num_try_send:
# since not all results got returned, raise rate limit error
raise ValueError("No deployments available for selected model")
raise ExpectNoException
with pytest.raises(expected_exception) as excinfo: # asserts correct type raised
_send_and_check()
print(expected_exception, excinfo)
if expected_exception is ValueError:
assert "No deployments available for selected model" in str(excinfo.value)
else:
assert len([i for i in results if i is not None]) == num_try_send
assert len([i for i in received if i is not None]) == num_try_send