From a112ba5f63dd9db389862b5ea46e12866a681ced Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 20 Aug 2026 19:36:26 -0700 Subject: [PATCH] test: enforce PT012 so a pytest.raises block cannot hide dead assertions (#37748) * test: enforce PT012 so a pytest.raises block cannot hide dead assertions `with pytest.raises(...)` stops at the first statement that raises. Anything sequenced after it inside the block never runs, so an assertion written there is never checked and the test still reports green. Two sites were doing exactly that, and both assertions turned out to be wrong once they started running. tests/llm_translation/test_prompt_factory.py asserted the bedrock rejection names "requires at least one non-system message", which holds. tests/proxy_unit_tests/test_proxy_server.py asserted the prisma startup failure mentions "httpx.ConnectError", which never appears: the failure is an httpx.ConnectError whose message is "All connection attempts failed", so that test now asserts the type. Its DATABASE_URL override moves to monkeypatch, since the old restore sat below the assertion and leaked the invalid URL into every later DB test the moment the assertion started being able to fail. The remaining 72 sites are rewritten without changing what they exercise: setup that cannot raise moves above the block, a nested `patch` moves outside it, and bodies with real control flow (a stream drain, an if/else on sync_mode, a retry loop) move into a local closure the block calls. Fixing PT012 unmasked two B017s, since ruff only reports a blind pytest.raises(Exception) once the block holds a single statement. tests/proxy_unit_tests/test_auth_checks.py narrows to the ProxyException can_key_call_model actually raises. tests/local_testing/test_completion_cost.py was asserting vertex_ai/medlm-medium has no cost entry, which stopped being true at some point; that dead first half is gone and the rest of the test, which checks medlm pricing resolves above zero, now runs instead of being skipped. * chore(ci): ratchet TQ004 to 768 after the prisma test moved to monkeypatch --- ruff-tests.toml | 5 ++- test-quality-budget.json | 2 +- .../test_bedrock_guardrails.py | 11 +++-- .../test_openai_responses_api.py | 5 ++- tests/llm_translation/test_prompt_factory.py | 3 +- tests/local_testing/test_aim_guardrails.py | 41 ++++++++++--------- tests/local_testing/test_completion_cost.py | 8 ---- tests/local_testing/test_exceptions.py | 5 ++- tests/local_testing/test_function_calling.py | 5 +-- tests/local_testing/test_mock_request.py | 3 +- .../test_router_budget_limiter.py | 12 ++---- tests/local_testing/test_router_fallbacks.py | 5 ++- .../test_router_max_parallel_requests.py | 5 ++- tests/local_testing/test_streaming.py | 7 +++- .../test_access_group_team_sync.py | 5 ++- tests/proxy_unit_tests/test_auth_checks.py | 6 +-- tests/proxy_unit_tests/test_jwt.py | 3 +- tests/proxy_unit_tests/test_proxy_server.py | 18 +++----- .../test_router_helper_utils.py | 14 ++++--- .../test_a2a_exception_mapping_utils.py | 7 +++- .../caching/test_redis_semantic_cache.py | 9 ++-- .../test_mcp_client.py | 7 ++-- .../bitbucket/test_bitbucket_integration.py | 31 ++++++++------ .../test_streaming_handler.py | 10 ++++- .../test_anthropic_chat_transformation.py | 6 ++- .../chat/test_bytez_chat_transformation.py | 7 ++-- .../custom_httpx/test_aiohttp_transport.py | 25 ++++++++--- .../test_credential_leak_prevention.py | 14 ++++--- .../oci/chat/test_oci_chat_transformation.py | 6 +-- .../llms/openai/test_openai_common_utils.py | 10 ++++- ...test_vertex_and_google_ai_studio_gemini.py | 5 ++- .../volcengine/test_volcengine_embedding.py | 7 ++-- .../test_async_streaming_error_propagation.py | 5 ++- .../passthrough/test_passthrough_main.py | 5 ++- ...test_streaming_interrupt_spend_tracking.py | 5 ++- .../proxy/auth/test_auth_exception_handler.py | 11 ++--- .../openai/test_moderations.py | 9 ++-- .../guardrail_hooks/test_cato_networks.py | 41 ++++++++++--------- .../guardrail_hooks/test_microsoft_purview.py | 15 +++++-- .../guardrail_hooks/test_panw_prisma_airs.py | 5 ++- .../test_prompt_security_guardrails.py | 6 +-- .../test_key_management_endpoints.py | 3 +- .../test_team_metadata_validation.py | 5 ++- .../proxy/test_budget_reservation.py | 15 +++++-- .../test_proxy_logging_hook_detection.py | 10 ++++- .../proxy/test_route_llm_request.py | 5 ++- .../repositories/test_unit_of_work.py | 10 ++++- .../test_streaming_iterator_error_events.py | 5 ++- .../test_custom_secret_manager.py | 8 ++-- tests/test_litellm/test_router.py | 15 +++++-- tests/test_ratelimit.py | 19 ++++++--- 51 files changed, 311 insertions(+), 193 deletions(-) diff --git a/ruff-tests.toml b/ruff-tests.toml index 8e90d6432df..6e77f4792a7 100644 --- a/ruff-tests.toml +++ b/ruff-tests.toml @@ -17,6 +17,9 @@ # B017 `pytest.raises(Exception)` accepts the TypeError a refactor introduced just as # readily as the rejection under test, so a crash reads as a pass. Narrow to the # real type, or add `match=` where the code genuinely raises a bare Exception +# PT012 a `pytest.raises` block that runs on past the raising call. Everything after +# that call is dead, so an `assert` sitting there is never checked. Keep the +# block to the call itself and put the assertions below it # # No target-version here on purpose: it resolves from requires-python (>=3.10), so # 3.11-only builtins like BaseExceptionGroup are correctly flagged in a tree that @@ -24,4 +27,4 @@ line-length = 120 -lint.select = ["F821", "B011", "B015", "B017", "B018", "PT015", "PLR0133", "PLW0127"] +lint.select = ["F821", "B011", "B015", "B017", "B018", "PT012", "PT015", "PLR0133", "PLW0127"] diff --git a/test-quality-budget.json b/test-quality-budget.json index fcb29c3191d..1613c8c75cb 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -9,7 +9,7 @@ "limit": 1078 }, "TQ004": { - "limit": 770 + "limit": 768 }, "TQ005": { "limit": 2832 diff --git a/tests/guardrails_tests/test_bedrock_guardrails.py b/tests/guardrails_tests/test_bedrock_guardrails.py index c7a5b79bbce..8b22cc0eb73 100644 --- a/tests/guardrails_tests/test_bedrock_guardrails.py +++ b/tests/guardrails_tests/test_bedrock_guardrails.py @@ -205,7 +205,7 @@ async def test_bedrock_guardrails_with_streaming(): mock_user_api_key_cache = MagicMock(spec=DualCache) mock_user_api_key_dict = UserAPIKeyAuth() - with pytest.raises(HTTPException): + async def _stream_through_guardrail(): proxy_logging_obj = ProxyLogging( user_api_key_cache=mock_user_api_key_cache, premium_user=True, @@ -240,6 +240,9 @@ async def test_bedrock_guardrails_with_streaming(): async for chunk in response: print(chunk) + with pytest.raises(HTTPException): + await _stream_through_guardrail() + @pytest.mark.asyncio async def test_bedrock_guardrails_with_streaming_no_violation(): @@ -1502,7 +1505,7 @@ async def test_bedrock_guardrail_disable_exception_on_block_streaming(): mock_post.return_value = mock_bedrock_response # Should raise exception during streaming processing - with pytest.raises(HTTPException): + async def _drain(): result_generator = ( guardrail_default.async_post_call_streaming_iterator_hook( user_api_key_dict=mock_user_api_key_dict, @@ -1511,10 +1514,12 @@ async def test_bedrock_guardrail_disable_exception_on_block_streaming(): ) ) - # Try to consume the generator - should raise exception async for chunk in result_generator: pass + with pytest.raises(HTTPException): + await _drain() + # Test 2: disable_exception_on_block=True. Streaming can't raise up to the # endpoint handler (SSE headers already flushed), so the block is delivered # as a synthetic stream with finish_reason=content_filter and the block diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index bd1517dbffb..d19fa09451c 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -1643,10 +1643,13 @@ async def test_openai_responses_api_token_limit_error(): model="gpt-5-mini", input=oversized_text, stream=True ) - with pytest.raises(litellm.APIError) as exc_info: + async def _drain(): async for event in response: print(event) + with pytest.raises(litellm.APIError) as exc_info: + await _drain() + assert exc_info.value.status_code == 400 assert "exceeds the context window" in str(exc_info.value) diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index ae215602e31..c3519fcb40f 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -1288,7 +1288,8 @@ def test_just_system_message(): model="anthropic.claude-3-sonnet-20240229-v1:0", llm_provider="bedrock", ) - assert "bedrock requires at least one non-system message" in str(e.value) + + assert "bedrock requires at least one non-system message" in str(e.value) def test_convert_generic_image_chunk_to_openai_image_obj(): diff --git a/tests/local_testing/test_aim_guardrails.py b/tests/local_testing/test_aim_guardrails.py index 2cb7f9cd357..5e5fb0d5459 100644 --- a/tests/local_testing/test_aim_guardrails.py +++ b/tests/local_testing/test_aim_guardrails.py @@ -101,26 +101,26 @@ async def test_block_callback(mode: str): ], } - with pytest.raises(ProxyException, match="Jailbreak detected") as exc_info: - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - return_value=Response( - json={ - "analysis_result": { - "analysis_time_ms": 212, - "policy_drill_down": {}, - "session_entities": [], - }, - "required_action": { - "action_type": "block_action", - "detection_message": "Jailbreak detected", - "policy_name": "blocking policy", - }, + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=Response( + json={ + "analysis_result": { + "analysis_time_ms": 212, + "policy_drill_down": {}, + "session_entities": [], }, - status_code=200, - request=Request(method="POST", url="http://aim"), - ), - ): + "required_action": { + "action_type": "block_action", + "detection_message": "Jailbreak detected", + "policy_name": "blocking policy", + }, + }, + status_code=200, + request=Request(method="POST", url="http://aim"), + ), + ): + async def _call_guardrail(): if mode == "pre_call": await aim_guardrail.async_pre_call_hook( data=data, @@ -135,6 +135,9 @@ async def test_block_callback(mode: str): call_type="completion", ) + with pytest.raises(ProxyException, match="Jailbreak detected") as exc_info: + await _call_guardrail() + exc = exc_info.value assert exc.code == "400" assert exc.type == "invalid_request_error" diff --git a/tests/local_testing/test_completion_cost.py b/tests/local_testing/test_completion_cost.py index e34d5c349c5..7dfcb55e29a 100644 --- a/tests/local_testing/test_completion_cost.py +++ b/tests/local_testing/test_completion_cost.py @@ -625,17 +625,9 @@ def test_vertex_ai_completion_cost(): print("calculated_input_cost: {}".format(calculated_input_cost)) -@pytest.mark.skip(reason="new test - WIP, working on fixing this") def test_vertex_ai_medlm_completion_cost(): """Test for medlm completion cost .""" - with pytest.raises(Exception) as e: - model = "vertex_ai/medlm-medium" - messages = [{"role": "user", "content": "Test MedLM completion cost."}] - predictive_cost = completion_cost( - model=model, messages=messages, custom_llm_provider="vertex_ai" - ) - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index 8c1df52e28e..edf847f4cef 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -1417,7 +1417,7 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model): import litellm litellm.set_verbose = True - with pytest.raises(Exception) as exc_info: + async def _call_with_bad_role(): if sync_mode: litellm.completion( model=model, @@ -1433,6 +1433,9 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model): sync_stream=sync_mode, ) + with pytest.raises(Exception) as exc_info: + await _call_with_bad_role() + assert exc_info.value.code == "invalid_value" assert exc_info.value.param is not None assert exc_info.value.type == "invalid_request_error" diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py index 4095962f91d..d6adde84400 100644 --- a/tests/local_testing/test_function_calling.py +++ b/tests/local_testing/test_function_calling.py @@ -357,14 +357,13 @@ def test_parallel_function_call_anthropic_error_msg( if expect_unsupported_params_error: with pytest.raises(litellm.UnsupportedParamsError) as e: - second_response = litellm.completion( + litellm.completion( model=model, messages=messages, temperature=0.2, seed=22, drop_params=True, - ) # get a new response from the model where it can see the function response - print("second response\n", second_response) + ) else: second_response = litellm.completion( model=model, diff --git a/tests/local_testing/test_mock_request.py b/tests/local_testing/test_mock_request.py index 710024b61b1..c9cd14633ba 100644 --- a/tests/local_testing/test_mock_request.py +++ b/tests/local_testing/test_mock_request.py @@ -128,13 +128,12 @@ def test_router_mock_request_with_mock_timeout(): ], ) with pytest.raises(litellm.Timeout): - response = router.completion( + router.completion( model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey, I'm a mock request"}], timeout=3, mock_timeout=True, ) - print(response) end_time = time.time() assert end_time - start_time >= 3, f"Time taken: {end_time - start_time}" diff --git a/tests/local_testing/test_router_budget_limiter.py b/tests/local_testing/test_router_budget_limiter.py index 1a36e9de8f2..48915137138 100644 --- a/tests/local_testing/test_router_budget_limiter.py +++ b/tests/local_testing/test_router_budget_limiter.py @@ -161,12 +161,10 @@ async def test_provider_budgets_e2e_test_expect_to_fail(): for _ in range(3): with pytest.raises(Exception) as exc_info: - response = await router.acompletion( + await router.acompletion( messages=[{"role": "user", "content": "Hello, how are you?"}], model="anthropic/claude-sonnet-4-5-20250929", ) - print(response) - print("response.hidden_params", response._hidden_params) await asyncio.sleep(0.5) # Verify the error is related to budget exceeded @@ -597,12 +595,10 @@ async def test_deployment_budgets_e2e_test_expect_to_fail(): for _ in range(3): with pytest.raises(Exception) as exc_info: - response = await router.acompletion( + await router.acompletion( messages=[{"role": "user", "content": "Hello, how are you?"}], model="openai/gpt-4o-mini", ) - print(response) - print("response.hidden_params", response._hidden_params) await asyncio.sleep(0.5) # Verify the error is related to budget exceeded @@ -651,13 +647,11 @@ async def test_tag_budgets_e2e_test_expect_to_fail(): for _ in range(3): with pytest.raises(Exception) as exc_info: - response = await router.acompletion( + await router.acompletion( messages=[{"role": "user", "content": "Hello, how are you?"}], model="openai/gpt-4o-mini", metadata={"tags": [TAG_NAME]}, ) - print(response) - print("response.hidden_params", response._hidden_params) await asyncio.sleep(0.5) # Verify the error is related to budget exceeded diff --git a/tests/local_testing/test_router_fallbacks.py b/tests/local_testing/test_router_fallbacks.py index 7c09c978029..15c6c5fec59 100644 --- a/tests/local_testing/test_router_fallbacks.py +++ b/tests/local_testing/test_router_fallbacks.py @@ -1416,7 +1416,7 @@ async def test_router_fallbacks_default_and_model_specific_fallbacks(sync_mode): default_fallbacks=["bad-model"], ) - with pytest.raises(Exception) as exc_info: + async def _call_bad_model(): if sync_mode: resp = router.completion( model="bad-model", @@ -1429,6 +1429,9 @@ async def test_router_fallbacks_default_and_model_specific_fallbacks(sync_mode): model="bad-model", messages=[{"role": "user", "content": "Hey, how's it going?"}], ) + + with pytest.raises(Exception) as exc_info: + await _call_bad_model() assert isinstance( exc_info.value, litellm.AuthenticationError ), f"Expected AuthenticationError, but got {type(exc_info.value).__name__}" diff --git a/tests/local_testing/test_router_max_parallel_requests.py b/tests/local_testing/test_router_max_parallel_requests.py index 1b81b9eb999..7bb40dd7a2f 100644 --- a/tests/local_testing/test_router_max_parallel_requests.py +++ b/tests/local_testing/test_router_max_parallel_requests.py @@ -205,9 +205,12 @@ async def test_max_parallel_requests_tpm_rate_limiting_base_case(): num_retries=0, ) - with pytest.raises(litellm.RateLimitError): + async def _exceed_limit(): for _ in range(2): await router.acompletion( model="gpt-4o-2024-08-06", messages=_messages, ) + + with pytest.raises(litellm.RateLimitError): + await _exceed_limit() diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index a4f564b227f..1fe9a1ab297 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -2926,11 +2926,14 @@ def test_unit_test_custom_stream_wrapper_repeating_chunk( print(f"expected_chunk_fail: {expected_chunk_fail}") if (loop_amount > litellm.REPEATED_STREAMING_CHUNK_LIMIT) and expected_chunk_fail: + def _drain(): + for chunk in response: + continue + with pytest.raises( (litellm.InternalServerError, litellm.exceptions.MidStreamFallbackError) ): - for chunk in response: - continue + _drain() else: for chunk in response: continue diff --git a/tests/proxy_admin_ui_tests/test_access_group_team_sync.py b/tests/proxy_admin_ui_tests/test_access_group_team_sync.py index 629d77f20fc..f7092d3ec00 100644 --- a/tests/proxy_admin_ui_tests/test_access_group_team_sync.py +++ b/tests/proxy_admin_ui_tests/test_access_group_team_sync.py @@ -170,12 +170,15 @@ async def test_a_failed_mirror_takes_the_new_team_row_with_it(): async with _clean_db() as db: await _seed(db, {GROUPS[0]: [], GROUPS[1]: [OTHER_TEAM]}) - with pytest.raises(RuntimeError): + async def _blow_up_after_reconcile(): async with db.tx() as tx: await tx.litellm_teamtable.create(data={"team_id": TEAM, "access_group_ids": [GROUPS[0]]}) await reconcile_team_access_group_membership(tx, TEAM) raise RuntimeError("the cache handoff blew up") + with pytest.raises(RuntimeError): + await _blow_up_after_reconcile() + assert await _read(db) == {GROUPS[0]: [], GROUPS[1]: [OTHER_TEAM]} assert await db.litellm_teamtable.find_unique(where={"team_id": TEAM}) is None diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/proxy_unit_tests/test_auth_checks.py index a7fd68def2a..ffcbe472be7 100644 --- a/tests/proxy_unit_tests/test_auth_checks.py +++ b/tests/proxy_unit_tests/test_auth_checks.py @@ -242,8 +242,8 @@ async def test_can_team_call_model(model, expect_to_work): ) @pytest.mark.asyncio async def test_can_key_call_model_wildcard_access(key_models, model, expect_to_work): + from litellm.proxy._types import ProxyException from litellm.proxy.auth.auth_checks import can_key_call_model - from fastapi import HTTPException llm_model_list = [ { @@ -294,7 +294,7 @@ async def test_can_key_call_model_wildcard_access(key_models, model, expect_to_w llm_router=router, ) else: - with pytest.raises(Exception) as e: + with pytest.raises(ProxyException): await can_key_call_model( model=model, llm_model_list=llm_model_list, @@ -302,8 +302,6 @@ async def test_can_key_call_model_wildcard_access(key_models, model, expect_to_w llm_router=router, ) - print(e) - @pytest.mark.parametrize( "key_models, model, expect_to_work", diff --git a/tests/proxy_unit_tests/test_jwt.py b/tests/proxy_unit_tests/test_jwt.py index a5836c59694..4db47a1cde4 100644 --- a/tests/proxy_unit_tests/test_jwt.py +++ b/tests/proxy_unit_tests/test_jwt.py @@ -1047,8 +1047,7 @@ async def test_allow_access_by_email( else: # Expect the call to fail with pytest.raises(ProxyException): - resp = await user_api_key_auth(request=request, api_key=bearer_token) - print(resp) + await user_api_key_auth(request=request, api_key=bearer_token) def test_get_public_key_from_jwk_url(): diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index bfbc92adc74..04bc80bf0d6 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -2412,12 +2412,14 @@ async def test_proxy_server_prisma_setup(): @pytest.mark.asyncio -async def test_proxy_server_prisma_setup_invalid_db(): +async def test_proxy_server_prisma_setup_invalid_db(monkeypatch): """ PROD TEST: Test that proxy server startup fails when it's unable to connect to the database Think 2-3 times before editing / deleting this test, it's important for PROD """ + import httpx + from litellm.proxy.proxy_server import ProxyStartupEvent from litellm.proxy.utils import ProxyLogging from litellm.caching import DualCache @@ -2425,24 +2427,14 @@ async def test_proxy_server_prisma_setup_invalid_db(): user_api_key_cache = DualCache() invalid_db_url = "postgresql://invalid:invalid@localhost:5432/nonexistent" - _old_db_url = os.getenv("DATABASE_URL") - os.environ["DATABASE_URL"] = invalid_db_url + monkeypatch.setenv("DATABASE_URL", invalid_db_url) - with pytest.raises(Exception) as exc_info: + with pytest.raises(httpx.ConnectError): await ProxyStartupEvent._setup_prisma_client( database_url=invalid_db_url, proxy_logging_obj=ProxyLogging(user_api_key_cache=user_api_key_cache), user_api_key_cache=user_api_key_cache, ) - print("GOT EXCEPTION=", exc_info) - - assert "httpx.ConnectError" in str(exc_info.value) - - # # Verify the error message indicates a database connection issue - # assert any(x in str(exc_info.value).lower() for x in ["database", "connection", "authentication"]) - - if _old_db_url: - os.environ["DATABASE_URL"] = _old_db_url @pytest.mark.asyncio diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index c3db9e67f9c..1fef0f01df8 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -423,10 +423,11 @@ def test_get_timeout(model_list): def test_handle_mock_testing_fallbacks(model_list, fallback_kwarg, expected_error): """Test if the '_handle_mock_testing_fallbacks' function is working correctly""" router = Router(model_list=model_list) + data = { + fallback_kwarg: True, + } + with pytest.raises(expected_error): - data = { - fallback_kwarg: True, - } router._handle_mock_testing_fallbacks( kwargs=data, ) @@ -435,10 +436,11 @@ def test_handle_mock_testing_fallbacks(model_list, fallback_kwarg, expected_erro def test_handle_mock_testing_rate_limit_error(model_list): """Test if the '_handle_mock_testing_rate_limit_error' function is working correctly""" router = Router(model_list=model_list) + data = { + "mock_testing_rate_limit_error": True, + } + with pytest.raises(litellm.RateLimitError): - data = { - "mock_testing_rate_limit_error": True, - } router._handle_mock_testing_rate_limit_error( kwargs=data, ) diff --git a/tests/test_litellm/a2a_protocol/test_a2a_exception_mapping_utils.py b/tests/test_litellm/a2a_protocol/test_a2a_exception_mapping_utils.py index 06191d1a370..c31d50960b1 100644 --- a/tests/test_litellm/a2a_protocol/test_a2a_exception_mapping_utils.py +++ b/tests/test_litellm/a2a_protocol/test_a2a_exception_mapping_utils.py @@ -171,9 +171,12 @@ async def test_stream_with_retry_raises_after_localhost_retries_exhausted(): api_base="https://agent.example", agent_name="test-agent", ) + async def _drain(): + async for _chunk in stream: + pytest.fail("expected retry exhaustion to raise before yielding") + with pytest.raises( RuntimeError, match="no response received after retry attempts", ): - async for _chunk in stream: - pytest.fail("expected retry exhaustion to raise before yielding") + await _drain() diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index 54f1fa721a2..66271579d31 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -310,11 +310,12 @@ def test_redis_semantic_cache_reraises_unexpected_isolated_index_error(monkeypat monkeypatch.setenv("REDIS_PORT", "6379") monkeypatch.setenv("REDIS_PASSWORD", "test_password") + cache = RedisSemanticCache( + similarity_threshold=0.8, + index_name="existing_index", + ) + with pytest.raises(ValueError, match="connection failed"): - cache = RedisSemanticCache( - similarity_threshold=0.8, - index_name="existing_index", - ) _ = cache.llmcache diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index 6c3c852395b..1ddb2cc1c8d 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -75,11 +75,10 @@ class TestMCPClient: # Test missing stdio_config client = MCPClient(transport_type=MCPTransport.stdio) + async def _noop(session): + return None + with pytest.raises(ValueError, match="stdio_config is required for stdio transport"): - - async def _noop(session): - return None - await client.run_with_session(_noop) @pytest.mark.asyncio diff --git a/tests/test_litellm/integrations/bitbucket/test_bitbucket_integration.py b/tests/test_litellm/integrations/bitbucket/test_bitbucket_integration.py index 46cd1d6e765..142be536f6b 100644 --- a/tests/test_litellm/integrations/bitbucket/test_bitbucket_integration.py +++ b/tests/test_litellm/integrations/bitbucket/test_bitbucket_integration.py @@ -88,39 +88,44 @@ def test_bitbucket_prompt_manager_error_handling(mock_client_class): "access_token": "test-token", } + manager = BitBucketPromptManager(config, prompt_id="test_prompt") + with pytest.raises( Exception, match="Failed to load prompt 'test_prompt' from BitBucket" ): - manager = BitBucketPromptManager(config, prompt_id="test_prompt") - _ = manager.prompt_manager # This triggers the error + _ = manager.prompt_manager def test_bitbucket_prompt_manager_config_validation(): """Test BitBucketPromptManager configuration validation.""" # Test missing required fields - validation happens when prompt_manager is accessed - with pytest.raises( - ValueError, match="workspace, repository, and access_token are required" - ): - manager = BitBucketPromptManager({}) - _ = manager.prompt_manager # This triggers validation + manager = BitBucketPromptManager({}) with pytest.raises( ValueError, match="workspace, repository, and access_token are required" ): - manager = BitBucketPromptManager({"workspace": "test"}) - _ = manager.prompt_manager # This triggers validation + _ = manager.prompt_manager + + manager = BitBucketPromptManager({"workspace": "test"}) with pytest.raises( ValueError, match="workspace, repository, and access_token are required" ): - manager = BitBucketPromptManager({"repository": "test"}) - _ = manager.prompt_manager # This triggers validation + _ = manager.prompt_manager + + manager = BitBucketPromptManager({"repository": "test"}) with pytest.raises( ValueError, match="workspace, repository, and access_token are required" ): - manager = BitBucketPromptManager({"access_token": "test"}) - _ = manager.prompt_manager # This triggers validation + _ = manager.prompt_manager + + manager = BitBucketPromptManager({"access_token": "test"}) + + with pytest.raises( + ValueError, match="workspace, repository, and access_token are required" + ): + _ = manager.prompt_manager @patch("litellm.integrations.bitbucket.bitbucket_prompt_manager.BitBucketClient") diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index fc7c81a9bab..bc457578e1e 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -2143,10 +2143,13 @@ def test_raise_on_model_repetition( chunks = _build_chunks(chunks_pattern, len(chunks_pattern)) if should_raise: - with pytest.raises(litellm.InternalServerError) as exc_info: + def _feed(): for chunk in chunks: wrapper.chunks.append(chunk) wrapper.raise_on_model_repetition() + + with pytest.raises(litellm.InternalServerError) as exc_info: + _feed() assert "repeating the same chunk" in str(exc_info.value) else: for chunk in chunks: @@ -3616,10 +3619,13 @@ async def test_transport_read_error_before_finish_reason_raises(logging_obj: Log ) received = [] - with pytest.raises(MidStreamFallbackError): + async def _drain(): async for chunk in response: received.append(chunk) + with pytest.raises(MidStreamFallbackError): + await _drain() + fabricated_finish_reasons = [ chunk.choices[0].finish_reason for chunk in received diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 391bd8566a2..d6aa384e03d 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1986,10 +1986,11 @@ def test_effort_validation(): ) assert result["output_config"]["effort"] == effort + optional_params = {"output_config": {"effort": "invalid"}} + with pytest.raises( litellm.exceptions.BadRequestError, match="Invalid effort value" ): - optional_params = {"output_config": {"effort": "invalid"}} config.transform_request( model="claude-opus-4-5-20251101", messages=messages, @@ -2043,11 +2044,12 @@ def test_max_effort_rejected_for_opus_45(): messages = [{"role": "user", "content": "Test"}] + optional_params = {"output_config": {"effort": "max"}} + with pytest.raises( litellm.exceptions.BadRequestError, match="effort='max' is not supported by this model", ): - optional_params = {"output_config": {"effort": "max"}} config.transform_request( model="claude-opus-4-5-20251101", messages=messages, diff --git a/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py b/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py index 2f8cc5484ba..e2421437720 100644 --- a/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py +++ b/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py @@ -35,11 +35,10 @@ class TestBytezChatConfig: assert result["user-agent"] == f"litellm/{version}" def test_missing_api_key(self): + config = BytezChatConfig() + headers = {} + with pytest.raises(Exception) as excinfo: - config = BytezChatConfig() - - headers = {} - config.validate_environment( headers=headers, model=TEST_MODEL, diff --git a/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py b/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py index 6a8cd29692f..2dc7fbfd62a 100644 --- a/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py +++ b/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py @@ -127,10 +127,13 @@ async def test_client_payload_error_mid_stream_raises_read_error(): stream = AiohttpResponseStream(mock_response) # type: ignore received_chunks = [] - with pytest.raises(httpx.ReadError): + async def _drain(): async for chunk in stream: received_chunks.append(chunk) + with pytest.raises(httpx.ReadError): + await _drain() + assert received_chunks == [b"chunk1"] assert mock_response.closed is True @@ -151,10 +154,13 @@ async def test_client_payload_error_before_first_chunk_raises_read_error(): stream = AiohttpResponseStream(mock_response) # type: ignore received_chunks = [] - with pytest.raises(httpx.ReadError): + async def _drain(): async for chunk in stream: received_chunks.append(chunk) + with pytest.raises(httpx.ReadError): + await _drain() + assert received_chunks == [] assert mock_response.closed is True @@ -171,10 +177,13 @@ async def test_connection_closed_runtime_error_raises_read_error(): stream = AiohttpResponseStream(mock_response) # type: ignore received_chunks = [] - with pytest.raises(httpx.ReadError): + async def _drain(): async for chunk in stream: received_chunks.append(chunk) + with pytest.raises(httpx.ReadError): + await _drain() + assert received_chunks == [b"data1"] assert mock_response.closed is True @@ -209,10 +218,13 @@ async def test_transfer_encoding_error_raises_read_error(): stream = AiohttpResponseStream(mock_response) # type: ignore received_chunks = [] - with pytest.raises(httpx.ReadError): + async def _drain(): async for chunk in stream: received_chunks.append(chunk) + with pytest.raises(httpx.ReadError): + await _drain() + assert received_chunks == [b"data1"] assert mock_response.closed is True @@ -254,10 +266,13 @@ async def test_timeout_exception_gets_mapped(): received_chunks = [] # This should raise httpx.TimeoutException (mapped from aiohttp.ServerTimeoutError) - with pytest.raises(httpx.TimeoutException): + async def _drain(): async for chunk in stream: received_chunks.append(chunk) + with pytest.raises(httpx.TimeoutException): + await _drain() + # Should have received the first chunk before the error assert received_chunks == [b"chunk1"] diff --git a/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py b/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py index 0a3bf403bf8..bd9db87a765 100644 --- a/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py +++ b/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py @@ -287,10 +287,11 @@ class TestHTTPHandlerErrorPaths: "send", side_effect=_make_httpx_status_error(url="https://api.test.com?key=SECRET"), ): + kwargs = {"url": "https://api.test.com?key=SECRET"} + if method != "delete": + kwargs["data"] = {"test": 1} + with pytest.raises(MaskedHTTPStatusError) as exc_info: - kwargs = {"url": "https://api.test.com?key=SECRET"} - if method != "delete": - kwargs["data"] = {"test": 1} getattr(sync_handler, method)(**kwargs) assert "SECRET" not in str(exc_info.value.request.url) @@ -304,10 +305,11 @@ class TestHTTPHandlerErrorPaths: new_callable=AsyncMock, side_effect=_make_httpx_status_error(url="https://api.test.com?key=SECRET"), ): + kwargs = {"url": "https://api.test.com?key=SECRET"} + if method != "delete": + kwargs["data"] = {"test": 1} + with pytest.raises(MaskedHTTPStatusError) as exc_info: - kwargs = {"url": "https://api.test.com?key=SECRET"} - if method != "delete": - kwargs["data"] = {"test": 1} await getattr(async_handler, method)(**kwargs) assert "SECRET" not in str(exc_info.value.request.url) diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py index 0f0033cae36..8be0780d86f 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -95,10 +95,10 @@ class TestOCIChatConfig: modified_params = params.copy() del modified_params[key] - with pytest.raises(Exception) as excinfo: - config = OCIChatConfig() - headers = {} + config = OCIChatConfig() + headers = {} + with pytest.raises(Exception) as excinfo: config.validate_environment( headers=headers, model=TEST_MODEL, diff --git a/tests/test_litellm/llms/openai/test_openai_common_utils.py b/tests/test_litellm/llms/openai/test_openai_common_utils.py index a28e133700e..bfd681cc06e 100644 --- a/tests/test_litellm/llms/openai/test_openai_common_utils.py +++ b/tests/test_litellm/llms/openai/test_openai_common_utils.py @@ -375,20 +375,26 @@ async def test_async_streaming_output_limit_400_maps_to_length_truncated_stream( @pytest.mark.parametrize("provider", ["openai", "azure"]) @pytest.mark.parametrize("stream", [False, True]) def test_sync_genuine_bad_request_still_raises(provider, stream): - with pytest.raises(litellm.BadRequestError): + def _call_and_drain(): result = litellm.completion( **_completion_kwargs(provider, _sync_client_raising(provider, GENUINE_400_MESSAGE), stream=stream) ) list(result) + with pytest.raises(litellm.BadRequestError): + _call_and_drain() + @pytest.mark.parametrize("provider", ["openai", "azure"]) @pytest.mark.parametrize("stream", [False, True]) @pytest.mark.asyncio async def test_async_genuine_bad_request_still_raises(provider, stream): - with pytest.raises(litellm.BadRequestError): + async def _call_and_drain(): result = await litellm.acompletion( **_completion_kwargs(provider, _async_client_raising(provider, GENUINE_400_MESSAGE), stream=stream) ) async for _ in result: pass + + with pytest.raises(litellm.BadRequestError): + await _call_and_drain() diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 51cc2857252..b7265ed62e9 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -5246,11 +5246,14 @@ def test_mid_stream_429_error_raises_during_iteration(): # Iterate the stream: first chunks should succeed, then 429 error should be raised results = [] - with pytest.raises(VertexAIError) as exc_info: + def _drain(): for chunk in streaming_obj: if chunk is not None: results.append(chunk) + with pytest.raises(VertexAIError) as exc_info: + _drain() + # Verify: received normal chunks before the error assert ( len(results) >= 1 diff --git a/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py b/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py index 6a035bcd7f0..07298f03f86 100644 --- a/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py +++ b/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py @@ -198,10 +198,11 @@ def test_volcengine_embedding_error_scenarios(): mock_embedding.side_effect = ValueError("Unsupported encoding_format") # Test that errors are properly raised + test_params = { + k: v for k, v in scenario.items() if k != "expected_error_pattern" + } + with pytest.raises(Exception) as exc_info: - test_params = { - k: v for k, v in scenario.items() if k != "expected_error_pattern" - } litellm.embedding(input=["test"], **test_params) # Verify error message contains expected pattern diff --git a/tests/test_litellm/passthrough/test_async_streaming_error_propagation.py b/tests/test_litellm/passthrough/test_async_streaming_error_propagation.py index d262063584b..faf4ea46c43 100644 --- a/tests/test_litellm/passthrough/test_async_streaming_error_propagation.py +++ b/tests/test_litellm/passthrough/test_async_streaming_error_propagation.py @@ -65,7 +65,7 @@ async def test_async_streaming_429_raises(): return mock_response chunks = [] - with pytest.raises(httpx.HTTPStatusError) as exc_info: + async def _drain(): async for chunk in _async_streaming( response=response_coro(), litellm_logging_obj=_make_mock_logging_obj(), @@ -73,6 +73,9 @@ async def test_async_streaming_429_raises(): ): chunks.append(chunk) + with pytest.raises(httpx.HTTPStatusError) as exc_info: + await _drain() + assert exc_info.value.response.status_code == 429 assert len(chunks) == 0 diff --git a/tests/test_litellm/passthrough/test_passthrough_main.py b/tests/test_litellm/passthrough/test_passthrough_main.py index 58a0185ea8c..965f9fd8f7d 100644 --- a/tests/test_litellm/passthrough/test_passthrough_main.py +++ b/tests/test_litellm/passthrough/test_passthrough_main.py @@ -721,10 +721,13 @@ async def test_allm_passthrough_route_429_streaming_raises(): # result is an async generator — consuming it must raise, not silently yield error bytes chunks = [] - with pytest.raises(httpx.HTTPStatusError) as exc_info: + async def _drain(): async for chunk in result: # type: ignore[union-attr] chunks.append(chunk) + with pytest.raises(httpx.HTTPStatusError) as exc_info: + await _drain() + assert exc_info.value.response.status_code == 429 assert len(chunks) == 0, "No chunks should be yielded before the 429 raises" diff --git a/tests/test_litellm/passthrough/test_streaming_interrupt_spend_tracking.py b/tests/test_litellm/passthrough/test_streaming_interrupt_spend_tracking.py index f3fe3ae5c38..3783e218e4e 100644 --- a/tests/test_litellm/passthrough/test_streaming_interrupt_spend_tracking.py +++ b/tests/test_litellm/passthrough/test_streaming_interrupt_spend_tracking.py @@ -164,7 +164,7 @@ async def test_async_streaming_flushes_on_upstream_exception_with_partial_data() provider_config = MagicMock() received = [] - with pytest.raises(httpx.ReadError): + async def _drain(): async for chunk in _async_streaming( response=response_coro(), litellm_logging_obj=mock_logging_obj, @@ -172,6 +172,9 @@ async def test_async_streaming_flushes_on_upstream_exception_with_partial_data() ): received.append(chunk) + with pytest.raises(httpx.ReadError): + await _drain() + assert received == partial_chunks await asyncio.sleep(0) diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index b4725a81823..721857e5411 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -338,12 +338,13 @@ async def test_handle_authentication_error_budget_exceeded(): mock_api_key = "test-key" # Test with budget exceeded error - with pytest.raises(ProxyException) as exc_info: - from litellm.exceptions import BudgetExceededError + from litellm.exceptions import BudgetExceededError - budget_error = BudgetExceededError( - message="Budget exceeded", current_cost=100, max_budget=100 - ) + budget_error = BudgetExceededError( + message="Budget exceeded", current_cost=100, max_budget=100 + ) + + with pytest.raises(ProxyException) as exc_info: await handler._handle_authentication_error( budget_error, mock_request, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py index 9002d1f81a3..729dcb54309 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py @@ -487,17 +487,18 @@ async def test_openai_moderation_guardrail_streaming_harmful_content(): # Should raise HTTPException when processing streaming harmful content from fastapi import HTTPException - with pytest.raises(HTTPException) as exc_info: + async def _drain(): result_chunks = [] - async for ( - chunk - ) in unified_guardrail.async_post_call_streaming_iterator_hook( + async for chunk in unified_guardrail.async_post_call_streaming_iterator_hook( user_api_key_dict=user_api_key_dict, response=mock_stream(), request_data=request_data, ): result_chunks.append(chunk) + with pytest.raises(HTTPException) as exc_info: + await _drain() + assert exc_info.value.status_code == 400 assert "Violated OpenAI moderation policy" in str(exc_info.value.detail) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cato_networks.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cato_networks.py index 428f2faf041..c23fbc0234e 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cato_networks.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cato_networks.py @@ -93,26 +93,26 @@ async def test_block_callback(mode: str): ], } - with pytest.raises(HTTPException, match="Jailbreak detected"): - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - return_value=Response( - json={ - "analysis_result": { - "analysis_time_ms": 212, - "policy_drill_down": {}, - "session_entities": [], - }, - "required_action": { - "action_type": "block_action", - "detection_message": "Jailbreak detected", - "policy_name": "blocking policy", - }, + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=Response( + json={ + "analysis_result": { + "analysis_time_ms": 212, + "policy_drill_down": {}, + "session_entities": [], }, - status_code=200, - request=Request(method="POST", url="http://cato"), - ), - ): + "required_action": { + "action_type": "block_action", + "detection_message": "Jailbreak detected", + "policy_name": "blocking policy", + }, + }, + status_code=200, + request=Request(method="POST", url="http://cato"), + ), + ): + async def _call_guardrail(): if mode == "pre_call": await cato_guardrail.async_pre_call_hook( data=data, @@ -127,6 +127,9 @@ async def test_block_callback(mode: str): call_type="completion", ) + with pytest.raises(HTTPException, match="Jailbreak detected"): + await _call_guardrail() + @pytest.mark.asyncio @pytest.mark.parametrize("mode", ["pre_call", "during_call"]) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py index cc89cea58d2..4a7a14fceaa 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py @@ -2441,7 +2441,7 @@ class TestStreamingIteratorHook: ), ): chunks = [] - with pytest.raises(HTTPException) as exc_info: + async def _drain(): async for chunk in guardrail.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth( api_key="test", user_id="user-123" @@ -2451,6 +2451,9 @@ class TestStreamingIteratorHook: ): chunks.append(chunk) + with pytest.raises(HTTPException) as exc_info: + await _drain() + assert exc_info.value.status_code == 400 assert len(chunks) == 0 # No chunks yielded before the block @@ -2477,7 +2480,7 @@ class TestStreamingIteratorHook: "litellm.main.stream_chunk_builder", return_value=assembled_response ): chunks = [] - with pytest.raises(HTTPException) as exc_info: + async def _drain(): async for chunk in guardrail.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth(api_key="test"), # no user_id response=fake_response_stream(), @@ -2485,6 +2488,9 @@ class TestStreamingIteratorHook: ): chunks.append(chunk) + with pytest.raises(HTTPException) as exc_info: + await _drain() + assert exc_info.value.status_code == 400 assert len(chunks) == 0 @@ -2625,7 +2631,7 @@ class TestStreamingIteratorHook: ), ): chunks = [] - with pytest.raises(HTTPException) as exc_info: + async def _drain(): async for chunk in guardrail.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth( api_key="test", user_id="user-123" @@ -2635,6 +2641,9 @@ class TestStreamingIteratorHook: ): chunks.append(chunk) + with pytest.raises(HTTPException) as exc_info: + await _drain() + assert exc_info.value.status_code == 400 assert len(chunks) == 0 diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py index 86a7ac1dabe..2284f2b678a 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py @@ -5902,7 +5902,7 @@ class TestPanwAirsBlockedErrorDetailPassthrough: with patch.object( base_handler, "_call_panw_api", return_value=copy.deepcopy(self._FULL_BLOCK_RESPONSE) ): - with pytest.raises(HTTPException) as exc_info: + async def _call_hook(): if is_response: await base_handler.async_post_call_success_hook( data=safe_prompt_data, @@ -5917,6 +5917,9 @@ class TestPanwAirsBlockedErrorDetailPassthrough: call_type="completion", ) + with pytest.raises(HTTPException) as exc_info: + await _call_hook() + error = exc_info.value.detail["error"] for field, value in self._FULL_BLOCK_RESPONSE.items(): if field == "category": diff --git a/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py b/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py index f35d64b89e3..c8f22e6c15e 100644 --- a/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py @@ -472,9 +472,9 @@ async def test_file_sanitization_block(): async def mock_get(*args, **kwargs): return mock_poll_response - with pytest.raises(HTTPException) as excinfo: - with patch.object(guardrail.async_handler, "post", side_effect=mock_post): - with patch.object(guardrail.async_handler, "get", side_effect=mock_get): + with patch.object(guardrail.async_handler, "post", side_effect=mock_post): + with patch.object(guardrail.async_handler, "get", side_effect=mock_get): + with pytest.raises(HTTPException) as excinfo: await guardrail.apply_guardrail( inputs=inputs, request_data=request_data, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 8e661af8daa..d4e9ccdca5e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -11037,8 +11037,7 @@ class TestKeyAliasSkipValidationOnUnchanged: assert new_alias != existing_alias with pytest.raises(ProxyException): - if new_alias != existing_alias: - _validate_key_alias_format(new_alias) + _validate_key_alias_format(new_alias) @pytest.mark.asyncio async def test_update_key_changed_to_valid_alias_passes( diff --git a/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py b/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py index 4c66f4aadf1..e4b031ade57 100644 --- a/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py +++ b/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py @@ -573,12 +573,15 @@ async def test_http_validator_service_outage_fails_closed(monkeypatch, kind, exi monkeypatch.setenv("TEAM_METADATA_VALIDATION_SERVICE_URL", _closed_port_url()) with _configured(impls.validate_via_http): - with pytest.raises(ProxyException) as exc_info: + async def _drive(): if kind == "create": await _drive_create(metadata=request_payload) else: await _drive_update(kind, existing_metadata, request_payload) + with pytest.raises(ProxyException) as exc_info: + await _drive() + assert str(exc_info.value.code) == "503" assert DEFAULT_TEAM_METADATA_VALIDATION_UNAVAILABLE_MESSAGE in str(exc_info.value.message) diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 133b53bb18d..2388654bf4b 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -2412,10 +2412,13 @@ async def test_streaming_cancel_before_any_chunk_reconciles_to_input_cost( generator, streaming_logging_obj = _drive_streaming_cancel(valid_token, cancel_before_chunk) received = [] - with pytest.raises(asyncio.CancelledError): + async def _drain(): async for chunk in generator: received.append(chunk) + with pytest.raises(asyncio.CancelledError): + await _drain() + assert received == [] # no chunk delivered, but the provider already received the input, so the # reservation is reconciled to the input cost (0.5), not refunded to zero @@ -2444,10 +2447,13 @@ async def test_streaming_cancel_after_chunk_keeps_reservation( generator, streaming_logging_obj = _drive_streaming_cancel(valid_token, cancel_after_chunk) received = [] - with pytest.raises(asyncio.CancelledError): + async def _drain(): async for chunk in generator: received.append(chunk) + with pytest.raises(asyncio.CancelledError): + await _drain() + assert received == ["data: chunk\n\n"] # a consumed stream must NOT be refunded assert counter_cache.in_memory_cache.get_cache( @@ -2508,10 +2514,13 @@ async def test_streaming_cancel_in_slow_path_before_yield_refunds(spend_counter_ received = [] # include_cost_in_streaming_usage forces fast_path off, so the hook above runs with patch.object(litellm, "include_cost_in_streaming_usage", True, create=True): - with pytest.raises(asyncio.CancelledError): + async def _drain(): async for chunk in generator: received.append(chunk) + with pytest.raises(asyncio.CancelledError): + await _drain() + assert received == [] # cancellation happened before any chunk reached the client, but the # provider already received the input -> reconcile to the input cost (0.5) diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py index 133156f9321..542572e1e56 100644 --- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py +++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py @@ -291,7 +291,7 @@ async def test_post_call_stream_guardrail_blocks_anthropic_messages_stream(monke yield chunk delivered = [] - with pytest.raises(HTTPException) as exc_info: + async def _drain(): async for chunk in proxy_logging.async_post_call_streaming_iterator_hook( response=fake_stream(), user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/messages"), @@ -299,6 +299,9 @@ async def test_post_call_stream_guardrail_blocks_anthropic_messages_stream(monke ): delivered.append(chunk) + with pytest.raises(HTTPException) as exc_info: + await _drain() + detail = exc_info.value.detail assert detail["guardrail_name"] == "output-filter" assert detail["keyword"] == "zebra" @@ -411,7 +414,7 @@ async def test_post_call_stream_guardrail_reroutes_inherited_apply_guardrail(mon yield chunk delivered = [] - with pytest.raises(HTTPException) as exc_info: + async def _drain(): async for chunk in proxy_logging.async_post_call_streaming_iterator_hook( response=fake_stream(), user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/messages"), @@ -419,6 +422,9 @@ async def test_post_call_stream_guardrail_reroutes_inherited_apply_guardrail(mon ): delivered.append(chunk) + with pytest.raises(HTTPException) as exc_info: + await _drain() + assert exc_info.value.detail["keyword"] == "zebra" assert delivered == [] diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index 1e716f7c148..23e0bbfb3ee 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -169,7 +169,7 @@ async def test_route_request_proxy_admin_can_call_all_team_scoped_deployments_wi ) ) - with pytest.raises(litellm.BadRequestError, match="multiple teams"): + async def _route_and_await(): ambiguous_call = await route_request( data=data, llm_router=router, @@ -179,6 +179,9 @@ async def test_route_request_proxy_admin_can_call_all_team_scoped_deployments_wi ) await ambiguous_call + with pytest.raises(litellm.BadRequestError, match="multiple teams"): + await _route_and_await() + router.add_deployment( Deployment( model_name="team-azure", diff --git a/tests/test_litellm/repositories/test_unit_of_work.py b/tests/test_litellm/repositories/test_unit_of_work.py index c270a570ad9..1ebfd917e36 100644 --- a/tests/test_litellm/repositories/test_unit_of_work.py +++ b/tests/test_litellm/repositories/test_unit_of_work.py @@ -59,11 +59,14 @@ async def test_updates_across_tables_share_one_batch_and_commit_once(): async def test_raising_inside_block_skips_commit(): batch = FakeBatch() - with pytest.raises(RuntimeError, match="boom"): + async def _blow_up_mid_transaction(): async with spend_reset_unit_of_work(lambda: batch) as uow: uow.keys.queue_spend_reset(token="tok-1", budget_reset_at=None) raise RuntimeError("boom") + with pytest.raises(RuntimeError, match="boom"): + await _blow_up_mid_transaction() + assert batch.commit_count == 0 @@ -119,9 +122,12 @@ async def test_budget_cascade_raising_inside_block_skips_commit(): the tier is still due on the next tick.""" batch = FakeBatch() - with pytest.raises(RuntimeError, match="boom"): + async def _blow_up_mid_transaction(): async with budget_cascade_unit_of_work(lambda: batch) as uow: uow.team_memberships.queue_spend_zero(where={"budget_id": {"in": ["budget-1"]}}) raise RuntimeError("boom") + with pytest.raises(RuntimeError, match="boom"): + await _blow_up_mid_transaction() + assert batch.commit_count == 0 diff --git a/tests/test_litellm/responses/test_streaming_iterator_error_events.py b/tests/test_litellm/responses/test_streaming_iterator_error_events.py index 3b87246ebdb..321abe4cc6d 100644 --- a/tests/test_litellm/responses/test_streaming_iterator_error_events.py +++ b/tests/test_litellm/responses/test_streaming_iterator_error_events.py @@ -208,9 +208,12 @@ async def test_async_iterator_error_after_first_chunk_carries_generated_content( ) chunks = [] - with pytest.raises(MidStreamFallbackError) as exc_info: + async def _drain(): async for chunk in iterator: chunks.append(chunk) + + with pytest.raises(MidStreamFallbackError) as exc_info: + await _drain() assert len(chunks) == 2 assert exc_info.value.status_code == 500 assert exc_info.value.is_pre_first_chunk is False diff --git a/tests/test_litellm/secret_managers/test_custom_secret_manager.py b/tests/test_litellm/secret_managers/test_custom_secret_manager.py index 1f4f9a47671..0426c5973cc 100644 --- a/tests/test_litellm/secret_managers/test_custom_secret_manager.py +++ b/tests/test_litellm/secret_managers/test_custom_secret_manager.py @@ -243,17 +243,17 @@ def test_minimal_custom_secret_manager(): assert value == "sync-TEST_KEY-value" # Write should raise NotImplementedError - with pytest.raises(NotImplementedError) as exc_info: - import asyncio + import asyncio + with pytest.raises(NotImplementedError) as exc_info: asyncio.run(secret_manager.async_write_secret("KEY", "value")) assert "Write operations are not implemented" in str(exc_info.value) # Delete should raise NotImplementedError - with pytest.raises(NotImplementedError) as exc_info: - import asyncio + import asyncio + with pytest.raises(NotImplementedError) as exc_info: asyncio.run(secret_manager.async_delete_secret("KEY")) assert "Delete operations are not implemented" in str(exc_info.value) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 9894fcef163..b50dc92c220 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1999,10 +1999,13 @@ async def test_acompletion_streaming_iterator(): # Collect streamed chunks — the first chunk succeeds, then the error re-raises collected_chunks = [] - with pytest.raises(MidStreamFallbackError): + async def _drain(): async for chunk in result: collected_chunks.append(chunk) + with pytest.raises(MidStreamFallbackError): + await _drain() + assert len(collected_chunks) == 1, "one chunk yielded before the error" print("✓ MidStreamFallbackError re-raised correctly when content was already generated") @@ -5557,10 +5560,13 @@ async def test_acompletion_streaming_iterator_does_not_log_success_on_terminal_f initial_kwargs=dict(initial_kwargs), ) collected = [] - with pytest.raises(MidStreamFallbackError): + async def _drain(): async for chunk in result: collected.append(chunk) + with pytest.raises(MidStreamFallbackError): + await _drain() + assert len(collected) == 1 logging_obj.dispatch_success_handlers.assert_not_called() @@ -5580,10 +5586,13 @@ async def test_acompletion_streaming_iterator_does_not_log_success_on_terminal_f initial_kwargs=dict(initial_kwargs), ) collected = [] - with pytest.raises(MidStreamFallbackError): + async def _drain(): async for chunk in result: collected.append(chunk) + with pytest.raises(MidStreamFallbackError): + await _drain() + assert len(collected) == 1, "only the partial chunk before the error" mock_fallback.assert_not_called() logging_obj.dispatch_success_handlers.assert_not_called() diff --git a/tests/test_ratelimit.py b/tests/test_ratelimit.py index 0469ded3f42..121dfbd99b7 100644 --- a/tests/test_ratelimit.py +++ b/tests/test_ratelimit.py @@ -149,19 +149,26 @@ def test_async_rate_limit( router: Router = router_factory(rpm, tpm, routing_strategy) print(f"router: {router.model_list}") - with pytest.raises(expected_exception) as excinfo: # asserts correct type raised - if sync_mode: - results = sync_call(router, list_of_messages) - else: - results = asyncio.run(async_call(router, list_of_messages)) + received = [] + + def _send_and_check(): + results = ( + sync_call(router, list_of_messages) + if sync_mode + else asyncio.run(async_call(router, list_of_messages)) + ) + received.extend(results) print(results) if len([i for i in results if i is not None]) != num_try_send: # since not all results got returned, raise rate limit error raise ValueError("No deployments available for selected model") raise ExpectNoException + with pytest.raises(expected_exception) as excinfo: # asserts correct type raised + _send_and_check() + print(expected_exception, excinfo) if expected_exception is ValueError: assert "No deployments available for selected model" in str(excinfo.value) else: - assert len([i for i in results if i is not None]) == num_try_send + assert len([i for i in received if i is not None]) == num_try_send