diff --git a/ruff-tests.toml b/ruff-tests.toml index 8e90d6432df..6e77f4792a7 100644 --- a/ruff-tests.toml +++ b/ruff-tests.toml @@ -17,6 +17,9 @@ # B017 `pytest.raises(Exception)` accepts the TypeError a refactor introduced just as # readily as the rejection under test, so a crash reads as a pass. Narrow to the # real type, or add `match=` where the code genuinely raises a bare Exception +# PT012 a `pytest.raises` block that runs on past the raising call. Everything after +# that call is dead, so an `assert` sitting there is never checked. Keep the +# block to the call itself and put the assertions below it # # No target-version here on purpose: it resolves from requires-python (>=3.10), so # 3.11-only builtins like BaseExceptionGroup are correctly flagged in a tree that @@ -24,4 +27,4 @@ line-length = 120 -lint.select = ["F821", "B011", "B015", "B017", "B018", "PT015", "PLR0133", "PLW0127"] +lint.select = ["F821", "B011", "B015", "B017", "B018", "PT012", "PT015", "PLR0133", "PLW0127"] diff --git a/test-quality-budget.json b/test-quality-budget.json index fcb29c3191d..1613c8c75cb 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -9,7 +9,7 @@ "limit": 1078 }, "TQ004": { - "limit": 770 + "limit": 768 }, "TQ005": { "limit": 2832 diff --git a/tests/guardrails_tests/test_bedrock_guardrails.py b/tests/guardrails_tests/test_bedrock_guardrails.py index c7a5b79bbce..8b22cc0eb73 100644 --- a/tests/guardrails_tests/test_bedrock_guardrails.py +++ b/tests/guardrails_tests/test_bedrock_guardrails.py @@ -205,7 +205,7 @@ async def test_bedrock_guardrails_with_streaming(): mock_user_api_key_cache = MagicMock(spec=DualCache) mock_user_api_key_dict = UserAPIKeyAuth() - with pytest.raises(HTTPException): + async def _stream_through_guardrail(): proxy_logging_obj = ProxyLogging( user_api_key_cache=mock_user_api_key_cache, premium_user=True, @@ -240,6 +240,9 @@ async def test_bedrock_guardrails_with_streaming(): async for chunk in response: print(chunk) + with pytest.raises(HTTPException): + await _stream_through_guardrail() + @pytest.mark.asyncio async def test_bedrock_guardrails_with_streaming_no_violation(): @@ -1502,7 +1505,7 @@ async def test_bedrock_guardrail_disable_exception_on_block_streaming(): mock_post.return_value = mock_bedrock_response # Should raise exception during streaming processing - with pytest.raises(HTTPException): + async def _drain(): result_generator = ( guardrail_default.async_post_call_streaming_iterator_hook( user_api_key_dict=mock_user_api_key_dict, @@ -1511,10 +1514,12 @@ async def test_bedrock_guardrail_disable_exception_on_block_streaming(): ) ) - # Try to consume the generator - should raise exception async for chunk in result_generator: pass + with pytest.raises(HTTPException): + await _drain() + # Test 2: disable_exception_on_block=True. Streaming can't raise up to the # endpoint handler (SSE headers already flushed), so the block is delivered # as a synthetic stream with finish_reason=content_filter and the block diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index bd1517dbffb..d19fa09451c 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -1643,10 +1643,13 @@ async def test_openai_responses_api_token_limit_error(): model="gpt-5-mini", input=oversized_text, stream=True ) - with pytest.raises(litellm.APIError) as exc_info: + async def _drain(): async for event in response: print(event) + with pytest.raises(litellm.APIError) as exc_info: + await _drain() + assert exc_info.value.status_code == 400 assert "exceeds the context window" in str(exc_info.value) diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index ae215602e31..c3519fcb40f 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -1288,7 +1288,8 @@ def test_just_system_message(): model="anthropic.claude-3-sonnet-20240229-v1:0", llm_provider="bedrock", ) - assert "bedrock requires at least one non-system message" in str(e.value) + + assert "bedrock requires at least one non-system message" in str(e.value) def test_convert_generic_image_chunk_to_openai_image_obj(): diff --git a/tests/local_testing/test_aim_guardrails.py b/tests/local_testing/test_aim_guardrails.py index 2cb7f9cd357..5e5fb0d5459 100644 --- a/tests/local_testing/test_aim_guardrails.py +++ b/tests/local_testing/test_aim_guardrails.py @@ -101,26 +101,26 @@ async def test_block_callback(mode: str): ], } - with pytest.raises(ProxyException, match="Jailbreak detected") as exc_info: - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - return_value=Response( - json={ - "analysis_result": { - "analysis_time_ms": 212, - "policy_drill_down": {}, - "session_entities": [], - }, - "required_action": { - "action_type": "block_action", - "detection_message": "Jailbreak detected", - "policy_name": "blocking policy", - }, + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=Response( + json={ + "analysis_result": { + "analysis_time_ms": 212, + "policy_drill_down": {}, + "session_entities": [], }, - status_code=200, - request=Request(method="POST", url="http://aim"), - ), - ): + "required_action": { + "action_type": "block_action", + "detection_message": "Jailbreak detected", + "policy_name": "blocking policy", + }, + }, + status_code=200, + request=Request(method="POST", url="http://aim"), + ), + ): + async def _call_guardrail(): if mode == "pre_call": await aim_guardrail.async_pre_call_hook( data=data, @@ -135,6 +135,9 @@ async def test_block_callback(mode: str): call_type="completion", ) + with pytest.raises(ProxyException, match="Jailbreak detected") as exc_info: + await _call_guardrail() + exc = exc_info.value assert exc.code == "400" assert exc.type == "invalid_request_error" diff --git a/tests/local_testing/test_completion_cost.py b/tests/local_testing/test_completion_cost.py index e34d5c349c5..7dfcb55e29a 100644 --- a/tests/local_testing/test_completion_cost.py +++ b/tests/local_testing/test_completion_cost.py @@ -625,17 +625,9 @@ def test_vertex_ai_completion_cost(): print("calculated_input_cost: {}".format(calculated_input_cost)) -@pytest.mark.skip(reason="new test - WIP, working on fixing this") def test_vertex_ai_medlm_completion_cost(): """Test for medlm completion cost .""" - with pytest.raises(Exception) as e: - model = "vertex_ai/medlm-medium" - messages = [{"role": "user", "content": "Test MedLM completion cost."}] - predictive_cost = completion_cost( - model=model, messages=messages, custom_llm_provider="vertex_ai" - ) - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index 8c1df52e28e..edf847f4cef 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -1417,7 +1417,7 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model): import litellm litellm.set_verbose = True - with pytest.raises(Exception) as exc_info: + async def _call_with_bad_role(): if sync_mode: litellm.completion( model=model, @@ -1433,6 +1433,9 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model): sync_stream=sync_mode, ) + with pytest.raises(Exception) as exc_info: + await _call_with_bad_role() + assert exc_info.value.code == "invalid_value" assert exc_info.value.param is not None assert exc_info.value.type == "invalid_request_error" diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py index 4095962f91d..d6adde84400 100644 --- a/tests/local_testing/test_function_calling.py +++ b/tests/local_testing/test_function_calling.py @@ -357,14 +357,13 @@ def test_parallel_function_call_anthropic_error_msg( if expect_unsupported_params_error: with pytest.raises(litellm.UnsupportedParamsError) as e: - second_response = litellm.completion( + litellm.completion( model=model, messages=messages, temperature=0.2, seed=22, drop_params=True, - ) # get a new response from the model where it can see the function response - print("second response\n", second_response) + ) else: second_response = litellm.completion( model=model, diff --git a/tests/local_testing/test_mock_request.py b/tests/local_testing/test_mock_request.py index 710024b61b1..c9cd14633ba 100644 --- a/tests/local_testing/test_mock_request.py +++ b/tests/local_testing/test_mock_request.py @@ -128,13 +128,12 @@ def test_router_mock_request_with_mock_timeout(): ], ) with pytest.raises(litellm.Timeout): - response = router.completion( + router.completion( model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey, I'm a mock request"}], timeout=3, mock_timeout=True, ) - print(response) end_time = time.time() assert end_time - start_time >= 3, f"Time taken: {end_time - start_time}" diff --git a/tests/local_testing/test_router_budget_limiter.py b/tests/local_testing/test_router_budget_limiter.py index 1a36e9de8f2..48915137138 100644 --- a/tests/local_testing/test_router_budget_limiter.py +++ b/tests/local_testing/test_router_budget_limiter.py @@ -161,12 +161,10 @@ async def test_provider_budgets_e2e_test_expect_to_fail(): for _ in range(3): with pytest.raises(Exception) as exc_info: - response = await router.acompletion( + await router.acompletion( messages=[{"role": "user", "content": "Hello, how are you?"}], model="anthropic/claude-sonnet-4-5-20250929", ) - print(response) - print("response.hidden_params", response._hidden_params) await asyncio.sleep(0.5) # Verify the error is related to budget exceeded @@ -597,12 +595,10 @@ async def test_deployment_budgets_e2e_test_expect_to_fail(): for _ in range(3): with pytest.raises(Exception) as exc_info: - response = await router.acompletion( + await router.acompletion( messages=[{"role": "user", "content": "Hello, how are you?"}], model="openai/gpt-4o-mini", ) - print(response) - print("response.hidden_params", response._hidden_params) await asyncio.sleep(0.5) # Verify the error is related to budget exceeded @@ -651,13 +647,11 @@ async def test_tag_budgets_e2e_test_expect_to_fail(): for _ in range(3): with pytest.raises(Exception) as exc_info: - response = await router.acompletion( + await router.acompletion( messages=[{"role": "user", "content": "Hello, how are you?"}], model="openai/gpt-4o-mini", metadata={"tags": [TAG_NAME]}, ) - print(response) - print("response.hidden_params", response._hidden_params) await asyncio.sleep(0.5) # Verify the error is related to budget exceeded diff --git a/tests/local_testing/test_router_fallbacks.py b/tests/local_testing/test_router_fallbacks.py index 7c09c978029..15c6c5fec59 100644 --- a/tests/local_testing/test_router_fallbacks.py +++ b/tests/local_testing/test_router_fallbacks.py @@ -1416,7 +1416,7 @@ async def test_router_fallbacks_default_and_model_specific_fallbacks(sync_mode): default_fallbacks=["bad-model"], ) - with pytest.raises(Exception) as exc_info: + async def _call_bad_model(): if sync_mode: resp = router.completion( model="bad-model", @@ -1429,6 +1429,9 @@ async def test_router_fallbacks_default_and_model_specific_fallbacks(sync_mode): model="bad-model", messages=[{"role": "user", "content": "Hey, how's it going?"}], ) + + with pytest.raises(Exception) as exc_info: + await _call_bad_model() assert isinstance( exc_info.value, litellm.AuthenticationError ), f"Expected AuthenticationError, but got {type(exc_info.value).__name__}" diff --git a/tests/local_testing/test_router_max_parallel_requests.py b/tests/local_testing/test_router_max_parallel_requests.py index 1b81b9eb999..7bb40dd7a2f 100644 --- a/tests/local_testing/test_router_max_parallel_requests.py +++ b/tests/local_testing/test_router_max_parallel_requests.py @@ -205,9 +205,12 @@ async def test_max_parallel_requests_tpm_rate_limiting_base_case(): num_retries=0, ) - with pytest.raises(litellm.RateLimitError): + async def _exceed_limit(): for _ in range(2): await router.acompletion( model="gpt-4o-2024-08-06", messages=_messages, ) + + with pytest.raises(litellm.RateLimitError): + await _exceed_limit() diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index a4f564b227f..1fe9a1ab297 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -2926,11 +2926,14 @@ def test_unit_test_custom_stream_wrapper_repeating_chunk( print(f"expected_chunk_fail: {expected_chunk_fail}") if (loop_amount > litellm.REPEATED_STREAMING_CHUNK_LIMIT) and expected_chunk_fail: + def _drain(): + for chunk in response: + continue + with pytest.raises( (litellm.InternalServerError, litellm.exceptions.MidStreamFallbackError) ): - for chunk in response: - continue + _drain() else: for chunk in response: continue diff --git a/tests/proxy_admin_ui_tests/test_access_group_team_sync.py b/tests/proxy_admin_ui_tests/test_access_group_team_sync.py index 629d77f20fc..f7092d3ec00 100644 --- a/tests/proxy_admin_ui_tests/test_access_group_team_sync.py +++ b/tests/proxy_admin_ui_tests/test_access_group_team_sync.py @@ -170,12 +170,15 @@ async def test_a_failed_mirror_takes_the_new_team_row_with_it(): async with _clean_db() as db: await _seed(db, {GROUPS[0]: [], GROUPS[1]: [OTHER_TEAM]}) - with pytest.raises(RuntimeError): + async def _blow_up_after_reconcile(): async with db.tx() as tx: await tx.litellm_teamtable.create(data={"team_id": TEAM, "access_group_ids": [GROUPS[0]]}) await reconcile_team_access_group_membership(tx, TEAM) raise RuntimeError("the cache handoff blew up") + with pytest.raises(RuntimeError): + await _blow_up_after_reconcile() + assert await _read(db) == {GROUPS[0]: [], GROUPS[1]: [OTHER_TEAM]} assert await db.litellm_teamtable.find_unique(where={"team_id": TEAM}) is None diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/proxy_unit_tests/test_auth_checks.py index a7fd68def2a..ffcbe472be7 100644 --- a/tests/proxy_unit_tests/test_auth_checks.py +++ b/tests/proxy_unit_tests/test_auth_checks.py @@ -242,8 +242,8 @@ async def test_can_team_call_model(model, expect_to_work): ) @pytest.mark.asyncio async def test_can_key_call_model_wildcard_access(key_models, model, expect_to_work): + from litellm.proxy._types import ProxyException from litellm.proxy.auth.auth_checks import can_key_call_model - from fastapi import HTTPException llm_model_list = [ { @@ -294,7 +294,7 @@ async def test_can_key_call_model_wildcard_access(key_models, model, expect_to_w llm_router=router, ) else: - with pytest.raises(Exception) as e: + with pytest.raises(ProxyException): await can_key_call_model( model=model, llm_model_list=llm_model_list, @@ -302,8 +302,6 @@ async def test_can_key_call_model_wildcard_access(key_models, model, expect_to_w llm_router=router, ) - print(e) - @pytest.mark.parametrize( "key_models, model, expect_to_work", diff --git a/tests/proxy_unit_tests/test_jwt.py b/tests/proxy_unit_tests/test_jwt.py index a5836c59694..4db47a1cde4 100644 --- a/tests/proxy_unit_tests/test_jwt.py +++ b/tests/proxy_unit_tests/test_jwt.py @@ -1047,8 +1047,7 @@ async def test_allow_access_by_email( else: # Expect the call to fail with pytest.raises(ProxyException): - resp = await user_api_key_auth(request=request, api_key=bearer_token) - print(resp) + await user_api_key_auth(request=request, api_key=bearer_token) def test_get_public_key_from_jwk_url(): diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index bfbc92adc74..04bc80bf0d6 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -2412,12 +2412,14 @@ async def test_proxy_server_prisma_setup(): @pytest.mark.asyncio -async def test_proxy_server_prisma_setup_invalid_db(): +async def test_proxy_server_prisma_setup_invalid_db(monkeypatch): """ PROD TEST: Test that proxy server startup fails when it's unable to connect to the database Think 2-3 times before editing / deleting this test, it's important for PROD """ + import httpx + from litellm.proxy.proxy_server import ProxyStartupEvent from litellm.proxy.utils import ProxyLogging from litellm.caching import DualCache @@ -2425,24 +2427,14 @@ async def test_proxy_server_prisma_setup_invalid_db(): user_api_key_cache = DualCache() invalid_db_url = "postgresql://invalid:invalid@localhost:5432/nonexistent" - _old_db_url = os.getenv("DATABASE_URL") - os.environ["DATABASE_URL"] = invalid_db_url + monkeypatch.setenv("DATABASE_URL", invalid_db_url) - with pytest.raises(Exception) as exc_info: + with pytest.raises(httpx.ConnectError): await ProxyStartupEvent._setup_prisma_client( database_url=invalid_db_url, proxy_logging_obj=ProxyLogging(user_api_key_cache=user_api_key_cache), user_api_key_cache=user_api_key_cache, ) - print("GOT EXCEPTION=", exc_info) - - assert "httpx.ConnectError" in str(exc_info.value) - - # # Verify the error message indicates a database connection issue - # assert any(x in str(exc_info.value).lower() for x in ["database", "connection", "authentication"]) - - if _old_db_url: - os.environ["DATABASE_URL"] = _old_db_url @pytest.mark.asyncio diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index c3db9e67f9c..1fef0f01df8 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -423,10 +423,11 @@ def test_get_timeout(model_list): def test_handle_mock_testing_fallbacks(model_list, fallback_kwarg, expected_error): """Test if the '_handle_mock_testing_fallbacks' function is working correctly""" router = Router(model_list=model_list) + data = { + fallback_kwarg: True, + } + with pytest.raises(expected_error): - data = { - fallback_kwarg: True, - } router._handle_mock_testing_fallbacks( kwargs=data, ) @@ -435,10 +436,11 @@ def test_handle_mock_testing_fallbacks(model_list, fallback_kwarg, expected_erro def test_handle_mock_testing_rate_limit_error(model_list): """Test if the '_handle_mock_testing_rate_limit_error' function is working correctly""" router = Router(model_list=model_list) + data = { + "mock_testing_rate_limit_error": True, + } + with pytest.raises(litellm.RateLimitError): - data = { - "mock_testing_rate_limit_error": True, - } router._handle_mock_testing_rate_limit_error( kwargs=data, ) diff --git a/tests/test_litellm/a2a_protocol/test_a2a_exception_mapping_utils.py b/tests/test_litellm/a2a_protocol/test_a2a_exception_mapping_utils.py index 06191d1a370..c31d50960b1 100644 --- a/tests/test_litellm/a2a_protocol/test_a2a_exception_mapping_utils.py +++ b/tests/test_litellm/a2a_protocol/test_a2a_exception_mapping_utils.py @@ -171,9 +171,12 @@ async def test_stream_with_retry_raises_after_localhost_retries_exhausted(): api_base="https://agent.example", agent_name="test-agent", ) + async def _drain(): + async for _chunk in stream: + pytest.fail("expected retry exhaustion to raise before yielding") + with pytest.raises( RuntimeError, match="no response received after retry attempts", ): - async for _chunk in stream: - pytest.fail("expected retry exhaustion to raise before yielding") + await _drain() diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index 54f1fa721a2..66271579d31 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -310,11 +310,12 @@ def test_redis_semantic_cache_reraises_unexpected_isolated_index_error(monkeypat monkeypatch.setenv("REDIS_PORT", "6379") monkeypatch.setenv("REDIS_PASSWORD", "test_password") + cache = RedisSemanticCache( + similarity_threshold=0.8, + index_name="existing_index", + ) + with pytest.raises(ValueError, match="connection failed"): - cache = RedisSemanticCache( - similarity_threshold=0.8, - index_name="existing_index", - ) _ = cache.llmcache diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index 6c3c852395b..1ddb2cc1c8d 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -75,11 +75,10 @@ class TestMCPClient: # Test missing stdio_config client = MCPClient(transport_type=MCPTransport.stdio) + async def _noop(session): + return None + with pytest.raises(ValueError, match="stdio_config is required for stdio transport"): - - async def _noop(session): - return None - await client.run_with_session(_noop) @pytest.mark.asyncio diff --git a/tests/test_litellm/integrations/bitbucket/test_bitbucket_integration.py b/tests/test_litellm/integrations/bitbucket/test_bitbucket_integration.py index 46cd1d6e765..142be536f6b 100644 --- a/tests/test_litellm/integrations/bitbucket/test_bitbucket_integration.py +++ b/tests/test_litellm/integrations/bitbucket/test_bitbucket_integration.py @@ -88,39 +88,44 @@ def test_bitbucket_prompt_manager_error_handling(mock_client_class): "access_token": "test-token", } + manager = BitBucketPromptManager(config, prompt_id="test_prompt") + with pytest.raises( Exception, match="Failed to load prompt 'test_prompt' from BitBucket" ): - manager = BitBucketPromptManager(config, prompt_id="test_prompt") - _ = manager.prompt_manager # This triggers the error + _ = manager.prompt_manager def test_bitbucket_prompt_manager_config_validation(): """Test BitBucketPromptManager configuration validation.""" # Test missing required fields - validation happens when prompt_manager is accessed - with pytest.raises( - ValueError, match="workspace, repository, and access_token are required" - ): - manager = BitBucketPromptManager({}) - _ = manager.prompt_manager # This triggers validation + manager = BitBucketPromptManager({}) with pytest.raises( ValueError, match="workspace, repository, and access_token are required" ): - manager = BitBucketPromptManager({"workspace": "test"}) - _ = manager.prompt_manager # This triggers validation + _ = manager.prompt_manager + + manager = BitBucketPromptManager({"workspace": "test"}) with pytest.raises( ValueError, match="workspace, repository, and access_token are required" ): - manager = BitBucketPromptManager({"repository": "test"}) - _ = manager.prompt_manager # This triggers validation + _ = manager.prompt_manager + + manager = BitBucketPromptManager({"repository": "test"}) with pytest.raises( ValueError, match="workspace, repository, and access_token are required" ): - manager = BitBucketPromptManager({"access_token": "test"}) - _ = manager.prompt_manager # This triggers validation + _ = manager.prompt_manager + + manager = BitBucketPromptManager({"access_token": "test"}) + + with pytest.raises( + ValueError, match="workspace, repository, and access_token are required" + ): + _ = manager.prompt_manager @patch("litellm.integrations.bitbucket.bitbucket_prompt_manager.BitBucketClient") diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index fc7c81a9bab..bc457578e1e 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -2143,10 +2143,13 @@ def test_raise_on_model_repetition( chunks = _build_chunks(chunks_pattern, len(chunks_pattern)) if should_raise: - with pytest.raises(litellm.InternalServerError) as exc_info: + def _feed(): for chunk in chunks: wrapper.chunks.append(chunk) wrapper.raise_on_model_repetition() + + with pytest.raises(litellm.InternalServerError) as exc_info: + _feed() assert "repeating the same chunk" in str(exc_info.value) else: for chunk in chunks: @@ -3616,10 +3619,13 @@ async def test_transport_read_error_before_finish_reason_raises(logging_obj: Log ) received = [] - with pytest.raises(MidStreamFallbackError): + async def _drain(): async for chunk in response: received.append(chunk) + with pytest.raises(MidStreamFallbackError): + await _drain() + fabricated_finish_reasons = [ chunk.choices[0].finish_reason for chunk in received diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 391bd8566a2..d6aa384e03d 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1986,10 +1986,11 @@ def test_effort_validation(): ) assert result["output_config"]["effort"] == effort + optional_params = {"output_config": {"effort": "invalid"}} + with pytest.raises( litellm.exceptions.BadRequestError, match="Invalid effort value" ): - optional_params = {"output_config": {"effort": "invalid"}} config.transform_request( model="claude-opus-4-5-20251101", messages=messages, @@ -2043,11 +2044,12 @@ def test_max_effort_rejected_for_opus_45(): messages = [{"role": "user", "content": "Test"}] + optional_params = {"output_config": {"effort": "max"}} + with pytest.raises( litellm.exceptions.BadRequestError, match="effort='max' is not supported by this model", ): - optional_params = {"output_config": {"effort": "max"}} config.transform_request( model="claude-opus-4-5-20251101", messages=messages, diff --git a/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py b/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py index 2f8cc5484ba..e2421437720 100644 --- a/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py +++ b/tests/test_litellm/llms/bytez/chat/test_bytez_chat_transformation.py @@ -35,11 +35,10 @@ class TestBytezChatConfig: assert result["user-agent"] == f"litellm/{version}" def test_missing_api_key(self): + config = BytezChatConfig() + headers = {} + with pytest.raises(Exception) as excinfo: - config = BytezChatConfig() - - headers = {} - config.validate_environment( headers=headers, model=TEST_MODEL, diff --git a/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py b/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py index 6a8cd29692f..2dc7fbfd62a 100644 --- a/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py +++ b/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py @@ -127,10 +127,13 @@ async def test_client_payload_error_mid_stream_raises_read_error(): stream = AiohttpResponseStream(mock_response) # type: ignore received_chunks = [] - with pytest.raises(httpx.ReadError): + async def _drain(): async for chunk in stream: received_chunks.append(chunk) + with pytest.raises(httpx.ReadError): + await _drain() + assert received_chunks == [b"chunk1"] assert mock_response.closed is True @@ -151,10 +154,13 @@ async def test_client_payload_error_before_first_chunk_raises_read_error(): stream = AiohttpResponseStream(mock_response) # type: ignore received_chunks = [] - with pytest.raises(httpx.ReadError): + async def _drain(): async for chunk in stream: received_chunks.append(chunk) + with pytest.raises(httpx.ReadError): + await _drain() + assert received_chunks == [] assert mock_response.closed is True @@ -171,10 +177,13 @@ async def test_connection_closed_runtime_error_raises_read_error(): stream = AiohttpResponseStream(mock_response) # type: ignore received_chunks = [] - with pytest.raises(httpx.ReadError): + async def _drain(): async for chunk in stream: received_chunks.append(chunk) + with pytest.raises(httpx.ReadError): + await _drain() + assert received_chunks == [b"data1"] assert mock_response.closed is True @@ -209,10 +218,13 @@ async def test_transfer_encoding_error_raises_read_error(): stream = AiohttpResponseStream(mock_response) # type: ignore received_chunks = [] - with pytest.raises(httpx.ReadError): + async def _drain(): async for chunk in stream: received_chunks.append(chunk) + with pytest.raises(httpx.ReadError): + await _drain() + assert received_chunks == [b"data1"] assert mock_response.closed is True @@ -254,10 +266,13 @@ async def test_timeout_exception_gets_mapped(): received_chunks = [] # This should raise httpx.TimeoutException (mapped from aiohttp.ServerTimeoutError) - with pytest.raises(httpx.TimeoutException): + async def _drain(): async for chunk in stream: received_chunks.append(chunk) + with pytest.raises(httpx.TimeoutException): + await _drain() + # Should have received the first chunk before the error assert received_chunks == [b"chunk1"] diff --git a/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py b/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py index 0a3bf403bf8..bd9db87a765 100644 --- a/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py +++ b/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py @@ -287,10 +287,11 @@ class TestHTTPHandlerErrorPaths: "send", side_effect=_make_httpx_status_error(url="https://api.test.com?key=SECRET"), ): + kwargs = {"url": "https://api.test.com?key=SECRET"} + if method != "delete": + kwargs["data"] = {"test": 1} + with pytest.raises(MaskedHTTPStatusError) as exc_info: - kwargs = {"url": "https://api.test.com?key=SECRET"} - if method != "delete": - kwargs["data"] = {"test": 1} getattr(sync_handler, method)(**kwargs) assert "SECRET" not in str(exc_info.value.request.url) @@ -304,10 +305,11 @@ class TestHTTPHandlerErrorPaths: new_callable=AsyncMock, side_effect=_make_httpx_status_error(url="https://api.test.com?key=SECRET"), ): + kwargs = {"url": "https://api.test.com?key=SECRET"} + if method != "delete": + kwargs["data"] = {"test": 1} + with pytest.raises(MaskedHTTPStatusError) as exc_info: - kwargs = {"url": "https://api.test.com?key=SECRET"} - if method != "delete": - kwargs["data"] = {"test": 1} await getattr(async_handler, method)(**kwargs) assert "SECRET" not in str(exc_info.value.request.url) diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py index 0f0033cae36..8be0780d86f 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -95,10 +95,10 @@ class TestOCIChatConfig: modified_params = params.copy() del modified_params[key] - with pytest.raises(Exception) as excinfo: - config = OCIChatConfig() - headers = {} + config = OCIChatConfig() + headers = {} + with pytest.raises(Exception) as excinfo: config.validate_environment( headers=headers, model=TEST_MODEL, diff --git a/tests/test_litellm/llms/openai/test_openai_common_utils.py b/tests/test_litellm/llms/openai/test_openai_common_utils.py index a28e133700e..bfd681cc06e 100644 --- a/tests/test_litellm/llms/openai/test_openai_common_utils.py +++ b/tests/test_litellm/llms/openai/test_openai_common_utils.py @@ -375,20 +375,26 @@ async def test_async_streaming_output_limit_400_maps_to_length_truncated_stream( @pytest.mark.parametrize("provider", ["openai", "azure"]) @pytest.mark.parametrize("stream", [False, True]) def test_sync_genuine_bad_request_still_raises(provider, stream): - with pytest.raises(litellm.BadRequestError): + def _call_and_drain(): result = litellm.completion( **_completion_kwargs(provider, _sync_client_raising(provider, GENUINE_400_MESSAGE), stream=stream) ) list(result) + with pytest.raises(litellm.BadRequestError): + _call_and_drain() + @pytest.mark.parametrize("provider", ["openai", "azure"]) @pytest.mark.parametrize("stream", [False, True]) @pytest.mark.asyncio async def test_async_genuine_bad_request_still_raises(provider, stream): - with pytest.raises(litellm.BadRequestError): + async def _call_and_drain(): result = await litellm.acompletion( **_completion_kwargs(provider, _async_client_raising(provider, GENUINE_400_MESSAGE), stream=stream) ) async for _ in result: pass + + with pytest.raises(litellm.BadRequestError): + await _call_and_drain() diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 51cc2857252..b7265ed62e9 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -5246,11 +5246,14 @@ def test_mid_stream_429_error_raises_during_iteration(): # Iterate the stream: first chunks should succeed, then 429 error should be raised results = [] - with pytest.raises(VertexAIError) as exc_info: + def _drain(): for chunk in streaming_obj: if chunk is not None: results.append(chunk) + with pytest.raises(VertexAIError) as exc_info: + _drain() + # Verify: received normal chunks before the error assert ( len(results) >= 1 diff --git a/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py b/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py index 6a035bcd7f0..07298f03f86 100644 --- a/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py +++ b/tests/test_litellm/llms/volcengine/test_volcengine_embedding.py @@ -198,10 +198,11 @@ def test_volcengine_embedding_error_scenarios(): mock_embedding.side_effect = ValueError("Unsupported encoding_format") # Test that errors are properly raised + test_params = { + k: v for k, v in scenario.items() if k != "expected_error_pattern" + } + with pytest.raises(Exception) as exc_info: - test_params = { - k: v for k, v in scenario.items() if k != "expected_error_pattern" - } litellm.embedding(input=["test"], **test_params) # Verify error message contains expected pattern diff --git a/tests/test_litellm/passthrough/test_async_streaming_error_propagation.py b/tests/test_litellm/passthrough/test_async_streaming_error_propagation.py index d262063584b..faf4ea46c43 100644 --- a/tests/test_litellm/passthrough/test_async_streaming_error_propagation.py +++ b/tests/test_litellm/passthrough/test_async_streaming_error_propagation.py @@ -65,7 +65,7 @@ async def test_async_streaming_429_raises(): return mock_response chunks = [] - with pytest.raises(httpx.HTTPStatusError) as exc_info: + async def _drain(): async for chunk in _async_streaming( response=response_coro(), litellm_logging_obj=_make_mock_logging_obj(), @@ -73,6 +73,9 @@ async def test_async_streaming_429_raises(): ): chunks.append(chunk) + with pytest.raises(httpx.HTTPStatusError) as exc_info: + await _drain() + assert exc_info.value.response.status_code == 429 assert len(chunks) == 0 diff --git a/tests/test_litellm/passthrough/test_passthrough_main.py b/tests/test_litellm/passthrough/test_passthrough_main.py index 58a0185ea8c..965f9fd8f7d 100644 --- a/tests/test_litellm/passthrough/test_passthrough_main.py +++ b/tests/test_litellm/passthrough/test_passthrough_main.py @@ -721,10 +721,13 @@ async def test_allm_passthrough_route_429_streaming_raises(): # result is an async generator — consuming it must raise, not silently yield error bytes chunks = [] - with pytest.raises(httpx.HTTPStatusError) as exc_info: + async def _drain(): async for chunk in result: # type: ignore[union-attr] chunks.append(chunk) + with pytest.raises(httpx.HTTPStatusError) as exc_info: + await _drain() + assert exc_info.value.response.status_code == 429 assert len(chunks) == 0, "No chunks should be yielded before the 429 raises" diff --git a/tests/test_litellm/passthrough/test_streaming_interrupt_spend_tracking.py b/tests/test_litellm/passthrough/test_streaming_interrupt_spend_tracking.py index f3fe3ae5c38..3783e218e4e 100644 --- a/tests/test_litellm/passthrough/test_streaming_interrupt_spend_tracking.py +++ b/tests/test_litellm/passthrough/test_streaming_interrupt_spend_tracking.py @@ -164,7 +164,7 @@ async def test_async_streaming_flushes_on_upstream_exception_with_partial_data() provider_config = MagicMock() received = [] - with pytest.raises(httpx.ReadError): + async def _drain(): async for chunk in _async_streaming( response=response_coro(), litellm_logging_obj=mock_logging_obj, @@ -172,6 +172,9 @@ async def test_async_streaming_flushes_on_upstream_exception_with_partial_data() ): received.append(chunk) + with pytest.raises(httpx.ReadError): + await _drain() + assert received == partial_chunks await asyncio.sleep(0) diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index b4725a81823..721857e5411 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -338,12 +338,13 @@ async def test_handle_authentication_error_budget_exceeded(): mock_api_key = "test-key" # Test with budget exceeded error - with pytest.raises(ProxyException) as exc_info: - from litellm.exceptions import BudgetExceededError + from litellm.exceptions import BudgetExceededError - budget_error = BudgetExceededError( - message="Budget exceeded", current_cost=100, max_budget=100 - ) + budget_error = BudgetExceededError( + message="Budget exceeded", current_cost=100, max_budget=100 + ) + + with pytest.raises(ProxyException) as exc_info: await handler._handle_authentication_error( budget_error, mock_request, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py index 9002d1f81a3..729dcb54309 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py @@ -487,17 +487,18 @@ async def test_openai_moderation_guardrail_streaming_harmful_content(): # Should raise HTTPException when processing streaming harmful content from fastapi import HTTPException - with pytest.raises(HTTPException) as exc_info: + async def _drain(): result_chunks = [] - async for ( - chunk - ) in unified_guardrail.async_post_call_streaming_iterator_hook( + async for chunk in unified_guardrail.async_post_call_streaming_iterator_hook( user_api_key_dict=user_api_key_dict, response=mock_stream(), request_data=request_data, ): result_chunks.append(chunk) + with pytest.raises(HTTPException) as exc_info: + await _drain() + assert exc_info.value.status_code == 400 assert "Violated OpenAI moderation policy" in str(exc_info.value.detail) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cato_networks.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cato_networks.py index 428f2faf041..c23fbc0234e 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cato_networks.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cato_networks.py @@ -93,26 +93,26 @@ async def test_block_callback(mode: str): ], } - with pytest.raises(HTTPException, match="Jailbreak detected"): - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - return_value=Response( - json={ - "analysis_result": { - "analysis_time_ms": 212, - "policy_drill_down": {}, - "session_entities": [], - }, - "required_action": { - "action_type": "block_action", - "detection_message": "Jailbreak detected", - "policy_name": "blocking policy", - }, + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + return_value=Response( + json={ + "analysis_result": { + "analysis_time_ms": 212, + "policy_drill_down": {}, + "session_entities": [], }, - status_code=200, - request=Request(method="POST", url="http://cato"), - ), - ): + "required_action": { + "action_type": "block_action", + "detection_message": "Jailbreak detected", + "policy_name": "blocking policy", + }, + }, + status_code=200, + request=Request(method="POST", url="http://cato"), + ), + ): + async def _call_guardrail(): if mode == "pre_call": await cato_guardrail.async_pre_call_hook( data=data, @@ -127,6 +127,9 @@ async def test_block_callback(mode: str): call_type="completion", ) + with pytest.raises(HTTPException, match="Jailbreak detected"): + await _call_guardrail() + @pytest.mark.asyncio @pytest.mark.parametrize("mode", ["pre_call", "during_call"]) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py index cc89cea58d2..4a7a14fceaa 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_microsoft_purview.py @@ -2441,7 +2441,7 @@ class TestStreamingIteratorHook: ), ): chunks = [] - with pytest.raises(HTTPException) as exc_info: + async def _drain(): async for chunk in guardrail.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth( api_key="test", user_id="user-123" @@ -2451,6 +2451,9 @@ class TestStreamingIteratorHook: ): chunks.append(chunk) + with pytest.raises(HTTPException) as exc_info: + await _drain() + assert exc_info.value.status_code == 400 assert len(chunks) == 0 # No chunks yielded before the block @@ -2477,7 +2480,7 @@ class TestStreamingIteratorHook: "litellm.main.stream_chunk_builder", return_value=assembled_response ): chunks = [] - with pytest.raises(HTTPException) as exc_info: + async def _drain(): async for chunk in guardrail.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth(api_key="test"), # no user_id response=fake_response_stream(), @@ -2485,6 +2488,9 @@ class TestStreamingIteratorHook: ): chunks.append(chunk) + with pytest.raises(HTTPException) as exc_info: + await _drain() + assert exc_info.value.status_code == 400 assert len(chunks) == 0 @@ -2625,7 +2631,7 @@ class TestStreamingIteratorHook: ), ): chunks = [] - with pytest.raises(HTTPException) as exc_info: + async def _drain(): async for chunk in guardrail.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth( api_key="test", user_id="user-123" @@ -2635,6 +2641,9 @@ class TestStreamingIteratorHook: ): chunks.append(chunk) + with pytest.raises(HTTPException) as exc_info: + await _drain() + assert exc_info.value.status_code == 400 assert len(chunks) == 0 diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py index 86a7ac1dabe..2284f2b678a 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py @@ -5902,7 +5902,7 @@ class TestPanwAirsBlockedErrorDetailPassthrough: with patch.object( base_handler, "_call_panw_api", return_value=copy.deepcopy(self._FULL_BLOCK_RESPONSE) ): - with pytest.raises(HTTPException) as exc_info: + async def _call_hook(): if is_response: await base_handler.async_post_call_success_hook( data=safe_prompt_data, @@ -5917,6 +5917,9 @@ class TestPanwAirsBlockedErrorDetailPassthrough: call_type="completion", ) + with pytest.raises(HTTPException) as exc_info: + await _call_hook() + error = exc_info.value.detail["error"] for field, value in self._FULL_BLOCK_RESPONSE.items(): if field == "category": diff --git a/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py b/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py index f35d64b89e3..c8f22e6c15e 100644 --- a/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/test_prompt_security_guardrails.py @@ -472,9 +472,9 @@ async def test_file_sanitization_block(): async def mock_get(*args, **kwargs): return mock_poll_response - with pytest.raises(HTTPException) as excinfo: - with patch.object(guardrail.async_handler, "post", side_effect=mock_post): - with patch.object(guardrail.async_handler, "get", side_effect=mock_get): + with patch.object(guardrail.async_handler, "post", side_effect=mock_post): + with patch.object(guardrail.async_handler, "get", side_effect=mock_get): + with pytest.raises(HTTPException) as excinfo: await guardrail.apply_guardrail( inputs=inputs, request_data=request_data, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 8e661af8daa..d4e9ccdca5e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -11037,8 +11037,7 @@ class TestKeyAliasSkipValidationOnUnchanged: assert new_alias != existing_alias with pytest.raises(ProxyException): - if new_alias != existing_alias: - _validate_key_alias_format(new_alias) + _validate_key_alias_format(new_alias) @pytest.mark.asyncio async def test_update_key_changed_to_valid_alias_passes( diff --git a/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py b/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py index 4c66f4aadf1..e4b031ade57 100644 --- a/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py +++ b/tests/test_litellm/proxy/management_helpers/test_team_metadata_validation.py @@ -573,12 +573,15 @@ async def test_http_validator_service_outage_fails_closed(monkeypatch, kind, exi monkeypatch.setenv("TEAM_METADATA_VALIDATION_SERVICE_URL", _closed_port_url()) with _configured(impls.validate_via_http): - with pytest.raises(ProxyException) as exc_info: + async def _drive(): if kind == "create": await _drive_create(metadata=request_payload) else: await _drive_update(kind, existing_metadata, request_payload) + with pytest.raises(ProxyException) as exc_info: + await _drive() + assert str(exc_info.value.code) == "503" assert DEFAULT_TEAM_METADATA_VALIDATION_UNAVAILABLE_MESSAGE in str(exc_info.value.message) diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 133b53bb18d..2388654bf4b 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -2412,10 +2412,13 @@ async def test_streaming_cancel_before_any_chunk_reconciles_to_input_cost( generator, streaming_logging_obj = _drive_streaming_cancel(valid_token, cancel_before_chunk) received = [] - with pytest.raises(asyncio.CancelledError): + async def _drain(): async for chunk in generator: received.append(chunk) + with pytest.raises(asyncio.CancelledError): + await _drain() + assert received == [] # no chunk delivered, but the provider already received the input, so the # reservation is reconciled to the input cost (0.5), not refunded to zero @@ -2444,10 +2447,13 @@ async def test_streaming_cancel_after_chunk_keeps_reservation( generator, streaming_logging_obj = _drive_streaming_cancel(valid_token, cancel_after_chunk) received = [] - with pytest.raises(asyncio.CancelledError): + async def _drain(): async for chunk in generator: received.append(chunk) + with pytest.raises(asyncio.CancelledError): + await _drain() + assert received == ["data: chunk\n\n"] # a consumed stream must NOT be refunded assert counter_cache.in_memory_cache.get_cache( @@ -2508,10 +2514,13 @@ async def test_streaming_cancel_in_slow_path_before_yield_refunds(spend_counter_ received = [] # include_cost_in_streaming_usage forces fast_path off, so the hook above runs with patch.object(litellm, "include_cost_in_streaming_usage", True, create=True): - with pytest.raises(asyncio.CancelledError): + async def _drain(): async for chunk in generator: received.append(chunk) + with pytest.raises(asyncio.CancelledError): + await _drain() + assert received == [] # cancellation happened before any chunk reached the client, but the # provider already received the input -> reconcile to the input cost (0.5) diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py index 133156f9321..542572e1e56 100644 --- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py +++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py @@ -291,7 +291,7 @@ async def test_post_call_stream_guardrail_blocks_anthropic_messages_stream(monke yield chunk delivered = [] - with pytest.raises(HTTPException) as exc_info: + async def _drain(): async for chunk in proxy_logging.async_post_call_streaming_iterator_hook( response=fake_stream(), user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/messages"), @@ -299,6 +299,9 @@ async def test_post_call_stream_guardrail_blocks_anthropic_messages_stream(monke ): delivered.append(chunk) + with pytest.raises(HTTPException) as exc_info: + await _drain() + detail = exc_info.value.detail assert detail["guardrail_name"] == "output-filter" assert detail["keyword"] == "zebra" @@ -411,7 +414,7 @@ async def test_post_call_stream_guardrail_reroutes_inherited_apply_guardrail(mon yield chunk delivered = [] - with pytest.raises(HTTPException) as exc_info: + async def _drain(): async for chunk in proxy_logging.async_post_call_streaming_iterator_hook( response=fake_stream(), user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/messages"), @@ -419,6 +422,9 @@ async def test_post_call_stream_guardrail_reroutes_inherited_apply_guardrail(mon ): delivered.append(chunk) + with pytest.raises(HTTPException) as exc_info: + await _drain() + assert exc_info.value.detail["keyword"] == "zebra" assert delivered == [] diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index 1e716f7c148..23e0bbfb3ee 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -169,7 +169,7 @@ async def test_route_request_proxy_admin_can_call_all_team_scoped_deployments_wi ) ) - with pytest.raises(litellm.BadRequestError, match="multiple teams"): + async def _route_and_await(): ambiguous_call = await route_request( data=data, llm_router=router, @@ -179,6 +179,9 @@ async def test_route_request_proxy_admin_can_call_all_team_scoped_deployments_wi ) await ambiguous_call + with pytest.raises(litellm.BadRequestError, match="multiple teams"): + await _route_and_await() + router.add_deployment( Deployment( model_name="team-azure", diff --git a/tests/test_litellm/repositories/test_unit_of_work.py b/tests/test_litellm/repositories/test_unit_of_work.py index c270a570ad9..1ebfd917e36 100644 --- a/tests/test_litellm/repositories/test_unit_of_work.py +++ b/tests/test_litellm/repositories/test_unit_of_work.py @@ -59,11 +59,14 @@ async def test_updates_across_tables_share_one_batch_and_commit_once(): async def test_raising_inside_block_skips_commit(): batch = FakeBatch() - with pytest.raises(RuntimeError, match="boom"): + async def _blow_up_mid_transaction(): async with spend_reset_unit_of_work(lambda: batch) as uow: uow.keys.queue_spend_reset(token="tok-1", budget_reset_at=None) raise RuntimeError("boom") + with pytest.raises(RuntimeError, match="boom"): + await _blow_up_mid_transaction() + assert batch.commit_count == 0 @@ -119,9 +122,12 @@ async def test_budget_cascade_raising_inside_block_skips_commit(): the tier is still due on the next tick.""" batch = FakeBatch() - with pytest.raises(RuntimeError, match="boom"): + async def _blow_up_mid_transaction(): async with budget_cascade_unit_of_work(lambda: batch) as uow: uow.team_memberships.queue_spend_zero(where={"budget_id": {"in": ["budget-1"]}}) raise RuntimeError("boom") + with pytest.raises(RuntimeError, match="boom"): + await _blow_up_mid_transaction() + assert batch.commit_count == 0 diff --git a/tests/test_litellm/responses/test_streaming_iterator_error_events.py b/tests/test_litellm/responses/test_streaming_iterator_error_events.py index 3b87246ebdb..321abe4cc6d 100644 --- a/tests/test_litellm/responses/test_streaming_iterator_error_events.py +++ b/tests/test_litellm/responses/test_streaming_iterator_error_events.py @@ -208,9 +208,12 @@ async def test_async_iterator_error_after_first_chunk_carries_generated_content( ) chunks = [] - with pytest.raises(MidStreamFallbackError) as exc_info: + async def _drain(): async for chunk in iterator: chunks.append(chunk) + + with pytest.raises(MidStreamFallbackError) as exc_info: + await _drain() assert len(chunks) == 2 assert exc_info.value.status_code == 500 assert exc_info.value.is_pre_first_chunk is False diff --git a/tests/test_litellm/secret_managers/test_custom_secret_manager.py b/tests/test_litellm/secret_managers/test_custom_secret_manager.py index 1f4f9a47671..0426c5973cc 100644 --- a/tests/test_litellm/secret_managers/test_custom_secret_manager.py +++ b/tests/test_litellm/secret_managers/test_custom_secret_manager.py @@ -243,17 +243,17 @@ def test_minimal_custom_secret_manager(): assert value == "sync-TEST_KEY-value" # Write should raise NotImplementedError - with pytest.raises(NotImplementedError) as exc_info: - import asyncio + import asyncio + with pytest.raises(NotImplementedError) as exc_info: asyncio.run(secret_manager.async_write_secret("KEY", "value")) assert "Write operations are not implemented" in str(exc_info.value) # Delete should raise NotImplementedError - with pytest.raises(NotImplementedError) as exc_info: - import asyncio + import asyncio + with pytest.raises(NotImplementedError) as exc_info: asyncio.run(secret_manager.async_delete_secret("KEY")) assert "Delete operations are not implemented" in str(exc_info.value) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 9894fcef163..b50dc92c220 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1999,10 +1999,13 @@ async def test_acompletion_streaming_iterator(): # Collect streamed chunks — the first chunk succeeds, then the error re-raises collected_chunks = [] - with pytest.raises(MidStreamFallbackError): + async def _drain(): async for chunk in result: collected_chunks.append(chunk) + with pytest.raises(MidStreamFallbackError): + await _drain() + assert len(collected_chunks) == 1, "one chunk yielded before the error" print("✓ MidStreamFallbackError re-raised correctly when content was already generated") @@ -5557,10 +5560,13 @@ async def test_acompletion_streaming_iterator_does_not_log_success_on_terminal_f initial_kwargs=dict(initial_kwargs), ) collected = [] - with pytest.raises(MidStreamFallbackError): + async def _drain(): async for chunk in result: collected.append(chunk) + with pytest.raises(MidStreamFallbackError): + await _drain() + assert len(collected) == 1 logging_obj.dispatch_success_handlers.assert_not_called() @@ -5580,10 +5586,13 @@ async def test_acompletion_streaming_iterator_does_not_log_success_on_terminal_f initial_kwargs=dict(initial_kwargs), ) collected = [] - with pytest.raises(MidStreamFallbackError): + async def _drain(): async for chunk in result: collected.append(chunk) + with pytest.raises(MidStreamFallbackError): + await _drain() + assert len(collected) == 1, "only the partial chunk before the error" mock_fallback.assert_not_called() logging_obj.dispatch_success_handlers.assert_not_called() diff --git a/tests/test_ratelimit.py b/tests/test_ratelimit.py index 0469ded3f42..121dfbd99b7 100644 --- a/tests/test_ratelimit.py +++ b/tests/test_ratelimit.py @@ -149,19 +149,26 @@ def test_async_rate_limit( router: Router = router_factory(rpm, tpm, routing_strategy) print(f"router: {router.model_list}") - with pytest.raises(expected_exception) as excinfo: # asserts correct type raised - if sync_mode: - results = sync_call(router, list_of_messages) - else: - results = asyncio.run(async_call(router, list_of_messages)) + received = [] + + def _send_and_check(): + results = ( + sync_call(router, list_of_messages) + if sync_mode + else asyncio.run(async_call(router, list_of_messages)) + ) + received.extend(results) print(results) if len([i for i in results if i is not None]) != num_try_send: # since not all results got returned, raise rate limit error raise ValueError("No deployments available for selected model") raise ExpectNoException + with pytest.raises(expected_exception) as excinfo: # asserts correct type raised + _send_and_check() + print(expected_exception, excinfo) if expected_exception is ValueError: assert "No deployments available for selected model" in str(excinfo.value) else: - assert len([i for i in results if i is not None]) == num_try_send + assert len([i for i in received if i is not None]) == num_try_send