test: unshadow the module handles the F811 sweep left behind (#37914)

* test: unshadow the module handles the F811 sweep left behind, and pin the two live tests that went red with it

The F811 sweep in #37878 removed the fixture-local `import litellm` from four
conftests, but the bare `import litellm.proxy.proxy_server` a few lines below
still binds `litellm` as a function local, so `importlib.reload(litellm)` runs
before the name is assigned and every test in those directories errors at
setup. The `hasattr` guard on the line above already proves the module is
loaded, so the import only ever bound the name. Drop it, and enable F823 in
ruff-tests.toml, which flags all four sites at the failing line and would have
blocked the sweep

The same sweep renamed the `check_non_streaming_response` parameter but left
one read of `completion`, which now resolves to `litellm.completion`, and
removed an import whose side effect was the only thing making
`litellm.proxy.proxy_server` reachable in the moderation hook test. That test
already takes `monkeypatch`, so patch the router through it and stop leaking
the router into later tests

`test_content_policy_exception_openai` passed vacuously until #37887 turned it
into a real `pytest.raises`, and OpenAI no longer rejects a lyrics prompt with
a content policy error. Inject an AsyncOpenAI client whose transport answers
with OpenAI's own `content_policy_violation` rejection so the mapping to
ContentPolicyViolationError is exercised every run

`test_async_create_batch` hit a 409 cancelling a batch OpenAI had already
marked failed. The cancel step tolerated a completed batch but not a failed
one. Fold both guards into one helper that tolerates a failed batch only when
OpenAI's recorded error is the org's enqueued token limit, and prints the
batch's errors so the reason is in the log either way

* test: close the injected AsyncOpenAI client after the content policy test

* chore(lint): ratchet TQ005 down by the global mutation this branch cleared

* chore(lint): ratchet TQ005 to 2660 on the merged tree

* chore(lint): ratchet TQ005 to 2561 on the merged tree

* chore(lint): ratchet TQ005 to 2548 on the merged tree

ruff-tests.toml

@@ -36,6 +36,10 @@
 #         `re.search`, so a `.` copied out of an error message is a wildcard and the block
 #         accepts messages the author never meant to accept. Mark a real regex raw, wrap a
 #         literal message in `re.escape`, and the pattern says which one it is
+# F823    a module-level name read inside a function that also binds it lower down. The
+#         later binding makes the name local for the whole body, so the read raises
+#         UnboundLocalError, and in an autouse fixture that takes every test in the
+#         directory down with it
 #
 # No target-version here on purpose: it resolves from requires-python (>=3.10), so
 # 3.11-only builtins like BaseExceptionGroup are correctly flagged in a tree that
@@ -58,4 +62,5 @@ lint.select = [
     "PLR0133",
     "PLW0127",
     "RUF043",
+    "F823",
 ]

test-quality-budget.json

@@ -12,7 +12,7 @@
     "limit": 469
   },
   "TQ005": {
-    "limit": 2406
+    "limit": 2405
   },
   "TQ006": {
     "limit": 34

tests/batches_tests/test_openai_batches_and_files.py

@@ -103,6 +103,25 @@ def load_vertex_ai_credentials():
     print("created gcs path service account=", os.environ["GCS_PATH_SERVICE_ACCOUNT"])
 
 
+async def cancel_batch_unless_already_terminal(batch_id: str, provider: str) -> None:
+    try:
+        cancel_batch_response = await litellm.acancel_batch(batch_id=batch_id, custom_llm_provider=provider)
+    except openai.ConflictError as e:
+        if "Cannot cancel a batch with status 'completed'" in str(e):
+            print(f"Batch already completed, cannot cancel: {e}")
+            return
+        if "Cannot cancel a batch with status 'failed'" not in str(e):
+            raise
+        failed_batch = await litellm.aretrieve_batch(batch_id=batch_id, custom_llm_provider=provider)
+        print(f"Batch failed before cancel, errors={failed_batch.errors}")
+        failure_codes = {err.code for err in (failed_batch.errors.data if failed_batch.errors else None) or []}
+        assert failure_codes == {"token_limit_exceeded"}, (
+            f"batch failed for a reason other than the org's enqueued token limit: {failed_batch.errors}"
+        )
+        return
+    print("cancel_batch_response=", cancel_batch_response)
+
+
 @pytest.mark.parametrize("provider", ["openai"])  # , "azure"
 @pytest.mark.asyncio
 @skip_if_no_openai_network
@@ -176,24 +195,7 @@ async def test_create_batch(provider, tmp_path):
     result_file_path = tmp_path / "batch_job_results_furniture.jsonl"
     result_file_path.write_bytes(result)
 
-    # Cancel Batch - handle race condition where batch may already be completed
-    try:
-        cancel_batch_response = await litellm.acancel_batch(
-            batch_id=create_batch_response.id,
-            custom_llm_provider=provider,
-        )
-        print("cancel_batch_response=", cancel_batch_response)
-    except openai.ConflictError as e:
-        # Only allow to pass if it's specifically the "batch already completed" error
-        if "Cannot cancel a batch with status 'completed'" in str(e):
-            print(f"Batch already completed, cannot cancel: {e}")
-        else:
-            # Re-raise other ConflictError types
-            raise
-    except Exception as e:
-        # Re-raise any other unexpected errors
-        print(f"Unexpected error during batch cancellation: {e}")
-        raise
+    await cancel_batch_unless_already_terminal(batch_id=create_batch_response.id, provider=provider)
 
     pass
 
@@ -395,24 +397,7 @@ async def test_async_create_batch(provider, tmp_path):
     result_file_path = tmp_path / "batch_job_results_furniture.jsonl"
     result_file_path.write_bytes(file_content.content)
 
-    # Cancel Batch - handle race condition where batch may already be completed
-    try:
-        cancel_batch_response = await litellm.acancel_batch(
-            batch_id=create_batch_response.id,
-            custom_llm_provider=provider,
-        )
-        print("cancel_batch_response=", cancel_batch_response)
-    except openai.ConflictError as e:
-        # Only allow to pass if it's specifically the "batch already completed" error
-        if "Cannot cancel a batch with status 'completed'" in str(e):
-            print(f"Batch already completed, cannot cancel: {e}")
-        else:
-            # Re-raise other ConflictError types
-            raise
-    except Exception as e:
-        # Re-raise any other unexpected errors
-        print(f"Unexpected error during batch cancellation: {e}")
-        raise
+    await cancel_batch_unless_already_terminal(batch_id=create_batch_response.id, provider=provider)
 
 
 mock_file_response = {

tests/enterprise/conftest.py

@@ -41,8 +41,6 @@ def setup_and_teardown():
 
     try:
         if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"):
-            import litellm.proxy.proxy_server
-
             importlib.reload(litellm.proxy.proxy_server)
     except Exception as e:
         print(f"Error reloading litellm.proxy.proxy_server: {e}")

tests/llm_responses_api_testing/conftest.py

@@ -86,8 +86,6 @@ def setup_and_teardown():
 
     try:
         if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"):
-            import litellm.proxy.proxy_server
-
             importlib.reload(litellm.proxy.proxy_server)
     except Exception as e:
         print(f"Error reloading litellm.proxy.proxy_server: {e}")

tests/local_testing/test_exceptions.py

@@ -5,7 +5,8 @@ import sys
 import traceback
 from typing import Any
 
-from openai import AuthenticationError, BadRequestError, OpenAIError, RateLimitError
+import httpx
+from openai import AsyncOpenAI, AuthenticationError, BadRequestError, OpenAIError, RateLimitError
 
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 
@@ -63,23 +64,38 @@ async def test_content_policy_exception_azure():
 
 @pytest.mark.asyncio
 async def test_content_policy_exception_openai():
-    # this is ony a test - we needed some way to invoke the exception :(
-    litellm.set_verbose = True
+    def reject_as_safety_system(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(
+            status_code=400,
+            json={
+                "error": {
+                    "message": "Your request was rejected as a result of our safety system.",
+                    "type": "invalid_request_error",
+                    "param": None,
+                    "code": "content_policy_violation",
+                }
+            },
+            request=request,
+        )
 
-    async def stream_response():
+    async def stream_response(rejecting_client: AsyncOpenAI):
         response = await litellm.acompletion(
             model="gpt-3.5-turbo",
             stream=True,
-            messages=[
-                {"role": "user", "content": "Gimme the lyrics to Don't Stop Me Now"}
-            ],
+            messages=[{"role": "user", "content": "Gimme the lyrics to Don't Stop Me Now"}],
+            client=rejecting_client,
         )
         async for chunk in response:
             print(chunk)
 
-    with pytest.raises(litellm.ContentPolicyViolationError) as exc_info:
-        await stream_response()
+    async with AsyncOpenAI(
+        api_key="sk-test",
+        http_client=httpx.AsyncClient(transport=httpx.MockTransport(reject_as_safety_system)),
+    ) as rejecting_client:
+        with pytest.raises(litellm.ContentPolicyViolationError) as exc_info:
+            await stream_response(rejecting_client)
     assert exc_info.value.llm_provider == "openai"
+    assert exc_info.value.status_code == 400
 
 
 # Test 1: Context Window Errors
@@ -871,7 +887,7 @@ def test_anthropic_tool_calling_exception():
 
 from typing import Optional, Union
 
-from openai import AsyncOpenAI, OpenAI
+from openai import OpenAI
 
 
 def _pre_call_utils(

tests/local_testing/test_openai_moderations_hook.py

@@ -62,7 +62,9 @@ async def test_openai_moderation_error_raising(monkeypatch):
 
     llm_router.amoderation = mock_amoderation
 
-    setattr(litellm.proxy.proxy_server, "llm_router", llm_router)
+    import litellm.proxy.proxy_server as proxy_server
+
+    monkeypatch.setattr(proxy_server, "llm_router", llm_router)
 
     with pytest.raises(Exception, match="Violated content safety policy") as exc_info:
         await openai_mod.async_moderation_hook(

tests/local_testing/test_stream_chunk_builder.py

@@ -15,7 +15,7 @@ def check_non_streaming_response(response):
     assert isinstance(
         response.choices[0].message.audio, ChatCompletionAudioResponse
     ), "Invalid audio response type"
-    assert len(completion.choices[0].message.audio.data) > 0, "Audio data is empty"
+    assert len(response.choices[0].message.audio.data) > 0, "Audio data is empty"
 
 
 sys.path.insert(

tests/router_unit_tests/conftest.py

@@ -58,8 +58,6 @@ def setup_and_teardown():
 
     try:
         if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"):
-            import litellm.proxy.proxy_server
-
             importlib.reload(litellm.proxy.proxy_server)
     except Exception as e:
         print(f"Error reloading litellm.proxy.proxy_server: {e}")

tests/vector_store_tests/conftest.py

@@ -28,8 +28,6 @@ def setup_and_teardown():
 
     try:
         if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"):
-            import litellm.proxy.proxy_server
-
             importlib.reload(litellm.proxy.proxy_server)
     except Exception as e:
         print(f"Error reloading litellm.proxy.proxy_server: {e}")