diff --git a/ruff-tests.toml b/ruff-tests.toml index de0931f5e69..e52e1a96d00 100644 --- a/ruff-tests.toml +++ b/ruff-tests.toml @@ -36,6 +36,10 @@ # `re.search`, so a `.` copied out of an error message is a wildcard and the block # accepts messages the author never meant to accept. Mark a real regex raw, wrap a # literal message in `re.escape`, and the pattern says which one it is +# F823 a module-level name read inside a function that also binds it lower down. The +# later binding makes the name local for the whole body, so the read raises +# UnboundLocalError, and in an autouse fixture that takes every test in the +# directory down with it # # No target-version here on purpose: it resolves from requires-python (>=3.10), so # 3.11-only builtins like BaseExceptionGroup are correctly flagged in a tree that @@ -58,4 +62,5 @@ lint.select = [ "PLR0133", "PLW0127", "RUF043", + "F823", ] diff --git a/test-quality-budget.json b/test-quality-budget.json index 28086b395f1..c16b34b9395 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -12,7 +12,7 @@ "limit": 469 }, "TQ005": { - "limit": 2406 + "limit": 2405 }, "TQ006": { "limit": 34 diff --git a/tests/batches_tests/test_openai_batches_and_files.py b/tests/batches_tests/test_openai_batches_and_files.py index 0a49b3d77d1..e849b087681 100644 --- a/tests/batches_tests/test_openai_batches_and_files.py +++ b/tests/batches_tests/test_openai_batches_and_files.py @@ -103,6 +103,25 @@ def load_vertex_ai_credentials(): print("created gcs path service account=", os.environ["GCS_PATH_SERVICE_ACCOUNT"]) +async def cancel_batch_unless_already_terminal(batch_id: str, provider: str) -> None: + try: + cancel_batch_response = await litellm.acancel_batch(batch_id=batch_id, custom_llm_provider=provider) + except openai.ConflictError as e: + if "Cannot cancel a batch with status 'completed'" in str(e): + print(f"Batch already completed, cannot cancel: {e}") + return + if "Cannot cancel a batch with status 'failed'" not in str(e): + raise + failed_batch = await litellm.aretrieve_batch(batch_id=batch_id, custom_llm_provider=provider) + print(f"Batch failed before cancel, errors={failed_batch.errors}") + failure_codes = {err.code for err in (failed_batch.errors.data if failed_batch.errors else None) or []} + assert failure_codes == {"token_limit_exceeded"}, ( + f"batch failed for a reason other than the org's enqueued token limit: {failed_batch.errors}" + ) + return + print("cancel_batch_response=", cancel_batch_response) + + @pytest.mark.parametrize("provider", ["openai"]) # , "azure" @pytest.mark.asyncio @skip_if_no_openai_network @@ -176,24 +195,7 @@ async def test_create_batch(provider, tmp_path): result_file_path = tmp_path / "batch_job_results_furniture.jsonl" result_file_path.write_bytes(result) - # Cancel Batch - handle race condition where batch may already be completed - try: - cancel_batch_response = await litellm.acancel_batch( - batch_id=create_batch_response.id, - custom_llm_provider=provider, - ) - print("cancel_batch_response=", cancel_batch_response) - except openai.ConflictError as e: - # Only allow to pass if it's specifically the "batch already completed" error - if "Cannot cancel a batch with status 'completed'" in str(e): - print(f"Batch already completed, cannot cancel: {e}") - else: - # Re-raise other ConflictError types - raise - except Exception as e: - # Re-raise any other unexpected errors - print(f"Unexpected error during batch cancellation: {e}") - raise + await cancel_batch_unless_already_terminal(batch_id=create_batch_response.id, provider=provider) pass @@ -395,24 +397,7 @@ async def test_async_create_batch(provider, tmp_path): result_file_path = tmp_path / "batch_job_results_furniture.jsonl" result_file_path.write_bytes(file_content.content) - # Cancel Batch - handle race condition where batch may already be completed - try: - cancel_batch_response = await litellm.acancel_batch( - batch_id=create_batch_response.id, - custom_llm_provider=provider, - ) - print("cancel_batch_response=", cancel_batch_response) - except openai.ConflictError as e: - # Only allow to pass if it's specifically the "batch already completed" error - if "Cannot cancel a batch with status 'completed'" in str(e): - print(f"Batch already completed, cannot cancel: {e}") - else: - # Re-raise other ConflictError types - raise - except Exception as e: - # Re-raise any other unexpected errors - print(f"Unexpected error during batch cancellation: {e}") - raise + await cancel_batch_unless_already_terminal(batch_id=create_batch_response.id, provider=provider) mock_file_response = { diff --git a/tests/enterprise/conftest.py b/tests/enterprise/conftest.py index f23a5664f83..8b23ba2998e 100644 --- a/tests/enterprise/conftest.py +++ b/tests/enterprise/conftest.py @@ -41,8 +41,6 @@ def setup_and_teardown(): try: if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - import litellm.proxy.proxy_server - importlib.reload(litellm.proxy.proxy_server) except Exception as e: print(f"Error reloading litellm.proxy.proxy_server: {e}") diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py index b5884f51275..72f70b9ead7 100644 --- a/tests/llm_responses_api_testing/conftest.py +++ b/tests/llm_responses_api_testing/conftest.py @@ -86,8 +86,6 @@ def setup_and_teardown(): try: if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - import litellm.proxy.proxy_server - importlib.reload(litellm.proxy.proxy_server) except Exception as e: print(f"Error reloading litellm.proxy.proxy_server: {e}") diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index cf89e7bea1d..bb96a1a84bb 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -5,7 +5,8 @@ import sys import traceback from typing import Any -from openai import AuthenticationError, BadRequestError, OpenAIError, RateLimitError +import httpx +from openai import AsyncOpenAI, AuthenticationError, BadRequestError, OpenAIError, RateLimitError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -63,23 +64,38 @@ async def test_content_policy_exception_azure(): @pytest.mark.asyncio async def test_content_policy_exception_openai(): - # this is ony a test - we needed some way to invoke the exception :( - litellm.set_verbose = True + def reject_as_safety_system(request: httpx.Request) -> httpx.Response: + return httpx.Response( + status_code=400, + json={ + "error": { + "message": "Your request was rejected as a result of our safety system.", + "type": "invalid_request_error", + "param": None, + "code": "content_policy_violation", + } + }, + request=request, + ) - async def stream_response(): + async def stream_response(rejecting_client: AsyncOpenAI): response = await litellm.acompletion( model="gpt-3.5-turbo", stream=True, - messages=[ - {"role": "user", "content": "Gimme the lyrics to Don't Stop Me Now"} - ], + messages=[{"role": "user", "content": "Gimme the lyrics to Don't Stop Me Now"}], + client=rejecting_client, ) async for chunk in response: print(chunk) - with pytest.raises(litellm.ContentPolicyViolationError) as exc_info: - await stream_response() + async with AsyncOpenAI( + api_key="sk-test", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(reject_as_safety_system)), + ) as rejecting_client: + with pytest.raises(litellm.ContentPolicyViolationError) as exc_info: + await stream_response(rejecting_client) assert exc_info.value.llm_provider == "openai" + assert exc_info.value.status_code == 400 # Test 1: Context Window Errors @@ -871,7 +887,7 @@ def test_anthropic_tool_calling_exception(): from typing import Optional, Union -from openai import AsyncOpenAI, OpenAI +from openai import OpenAI def _pre_call_utils( diff --git a/tests/local_testing/test_openai_moderations_hook.py b/tests/local_testing/test_openai_moderations_hook.py index 944ac047e55..4f98eb608fa 100644 --- a/tests/local_testing/test_openai_moderations_hook.py +++ b/tests/local_testing/test_openai_moderations_hook.py @@ -62,7 +62,9 @@ async def test_openai_moderation_error_raising(monkeypatch): llm_router.amoderation = mock_amoderation - setattr(litellm.proxy.proxy_server, "llm_router", llm_router) + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", llm_router) with pytest.raises(Exception, match="Violated content safety policy") as exc_info: await openai_mod.async_moderation_hook( diff --git a/tests/local_testing/test_stream_chunk_builder.py b/tests/local_testing/test_stream_chunk_builder.py index 9dab6e60c35..823983d9e2d 100644 --- a/tests/local_testing/test_stream_chunk_builder.py +++ b/tests/local_testing/test_stream_chunk_builder.py @@ -15,7 +15,7 @@ def check_non_streaming_response(response): assert isinstance( response.choices[0].message.audio, ChatCompletionAudioResponse ), "Invalid audio response type" - assert len(completion.choices[0].message.audio.data) > 0, "Audio data is empty" + assert len(response.choices[0].message.audio.data) > 0, "Audio data is empty" sys.path.insert( diff --git a/tests/router_unit_tests/conftest.py b/tests/router_unit_tests/conftest.py index db6a722a926..c759f9fa74c 100644 --- a/tests/router_unit_tests/conftest.py +++ b/tests/router_unit_tests/conftest.py @@ -58,8 +58,6 @@ def setup_and_teardown(): try: if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - import litellm.proxy.proxy_server - importlib.reload(litellm.proxy.proxy_server) except Exception as e: print(f"Error reloading litellm.proxy.proxy_server: {e}") diff --git a/tests/vector_store_tests/conftest.py b/tests/vector_store_tests/conftest.py index 41da685895b..48a82ea60a6 100644 --- a/tests/vector_store_tests/conftest.py +++ b/tests/vector_store_tests/conftest.py @@ -28,8 +28,6 @@ def setup_and_teardown(): try: if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - import litellm.proxy.proxy_server - importlib.reload(litellm.proxy.proxy_server) except Exception as e: print(f"Error reloading litellm.proxy.proxy_server: {e}")