Merge pull request #42103 from BerriAI/litellm_test_unit_ci_wiring

ci(tests): wire tests/unit into CircleCI and keep draining GHA shards green

.circleci/config.yml

@@ -3142,6 +3142,33 @@ jobs:
       - store_artifacts:
           path: test-results
 
+  unit:
+    machine:
+      image: ubuntu-2204:2024.04.1
+    resource_class: large
+    working_directory: ~/project
+    steps:
+      - setup_litellm_test_deps
+      - run:
+          name: Generate Prisma client
+          command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+      - run:
+          name: Run unit tests
+          command: |
+            mkdir -p test-results/unit
+            mapfile -t files < <(find tests/unit -name 'test_*.py' | sort)
+            if [ "${#files[@]}" -eq 0 ]; then echo "tests/unit holds no test_*.py files; nothing to run"; exit 0; fi
+            set +e
+            LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short --junitxml=test-results/unit/junit.xml
+            status=$?
+            set -e
+            if [ "$status" -eq 5 ]; then echo "pytest collected no tests from tests/unit; passing"; exit 0; fi
+            exit "$status"
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
+
 workflows:
   migration_startup:
     when: << pipeline.parameters.run_migration_tests >>
@@ -3190,6 +3217,8 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - unit:
+          filters: *main_branches
       - provider_replay_harness
       - base_sdk_install:
           filters: *main_branches

.github/workflows/_test-unit-base.yml

@@ -165,33 +165,41 @@ jobs:
           DIST: ${{ inputs.dist }}
           COVERAGE_CORE: sysmon
         run: |
-          if [ "${WORKERS}" = "0" ]; then
-            uv run --no-sync pytest ${TEST_PATH:?} \
-              --tb=short -vv \
-              --maxfail="${MAX_FAILURES}" \
-              --reruns "${RERUNS}" \
-              --reruns-delay 1 \
-              --timeout="${TEST_TIMEOUT_SECONDS}" \
-              --rerun-except "from pytest-timeout" \
-              --durations=20 \
-              --cov=./litellm --cov=./enterprise/litellm_enterprise \
-              --cov-report=xml:coverage.xml \
-              --cov-config=pyproject.toml
-          else
-            uv run --no-sync pytest ${TEST_PATH:?} \
-              --tb=short -vv \
-              --maxfail="${MAX_FAILURES}" \
-              -n "${WORKERS}" \
-              --reruns "${RERUNS}" \
-              --reruns-delay 1 \
-              --timeout="${TEST_TIMEOUT_SECONDS}" \
-              --rerun-except "from pytest-timeout" \
-              --dist="${DIST}" \
-              --durations=20 \
-              --cov=./litellm --cov=./enterprise/litellm_enterprise \
-              --cov-report=xml:coverage.xml \
-              --cov-config=pyproject.toml
+          found_path=false
+          for path in ${TEST_PATH}; do
+            if [ -e "${path%%::*}" ]; then
+              found_path=true
+              break
+            fi
+          done
+          if [ "$found_path" = false ]; then
+            echo "No path in TEST_PATH exists (${TEST_PATH}); nothing to run"
+            exit 0
           fi
+          xdist_args=()
+          if [ "${WORKERS}" != "0" ]; then
+            xdist_args=(-n "${WORKERS}" --dist="${DIST}")
+          fi
+          set +e
+          uv run --no-sync pytest ${TEST_PATH:?} \
+            --tb=short -vv \
+            --maxfail="${MAX_FAILURES}" \
+            "${xdist_args[@]}" \
+            --reruns "${RERUNS}" \
+            --reruns-delay 1 \
+            --timeout="${TEST_TIMEOUT_SECONDS}" \
+            --rerun-except "from pytest-timeout" \
+            --durations=20 \
+            --cov=./litellm --cov=./enterprise/litellm_enterprise \
+            --cov-report=xml:coverage.xml \
+            --cov-config=pyproject.toml
+          status=$?
+          set -e
+          if [ "$status" -eq 5 ]; then
+            echo "pytest collected no tests from ${TEST_PATH}; passing"
+            exit 0
+          fi
+          exit "$status"
 
       - name: Save coverage report
         if: always() && steps.changes.outputs.decision != 'skip'

pyproject.toml

@@ -196,6 +196,7 @@ dev = [
     "mypy==1.20.1",
     "keyring==25.7.0",
     "pytest==9.0.3",
+    "pytest-socket==0.8.1",
     "tomli==2.4.1; python_version < '3.11'",
     "pytest-mock==3.15.1",
     "pytest-asyncio==1.3.0",

scripts/check_test_quality.py

@@ -48,10 +48,6 @@ TQ006   A `pytest.skip` reached only when a credential-shaped environment variab
         deliberate branch. The gate follows one local or module-level binding, which is
         the `key = os.getenv(...)` then `if not key: pytest.skip(...)` shape most of
         these use.
-TQ008   A `patch(...)` whose target is a `litellm.` internal. Patching the SDK's own
-        functions pins the test to the current wiring instead of the behaviour, and it
-        is the idiom the suite reaches for instead of faking the HTTP boundary. Mocking
-        a third-party client, a transport, or anything outside `litellm.` is untouched.
 TQ007   A module global that a conftest saves before every test and restores after it.
         The save/restore list is a hand-maintained inventory of the leaks the suite
         already knows about, so it is allowed to shrink and never to grow: a new entry
@@ -483,67 +479,6 @@ def iter_global_mutation_violations(path: Path, tree: ast.Module) -> Iterator[Vi
                 )
 
 
-def _is_sdk_internal(dotted: str) -> bool:
-    return dotted == SDK_MODULE or dotted.startswith(f"{SDK_MODULE}.")
-
-
-def _sdk_import_bindings(tree: ast.Module) -> Iterator[tuple[str, str]]:
-    """(local name, dotted path) for every import that binds something under `litellm`."""
-    for node in ast.walk(tree):
-        if isinstance(node, ast.Import):
-            yield from (
-                (alias.asname, alias.name) if alias.asname else (root, root)
-                for alias in node.names
-                if _is_sdk_internal(alias.name)
-                for root in (alias.name.partition(".")[0],)
-            )
-        elif isinstance(node, ast.ImportFrom) and node.module and _is_sdk_internal(node.module):
-            yield from ((alias.asname or alias.name, f"{node.module}.{alias.name}") for alias in node.names)
-
-
-def _sdk_aliases(tree: ast.Module) -> Mapping[str, str]:
-    """Local names bound to something under `litellm`, mapped to the path they stand for.
-
-    `from litellm.llms.openai.chat import handler` then `patch.object(handler.X, ...)`
-    reaches the same internal as the dotted string form and has to read the same way.
-    """
-    return MappingProxyType({name: dotted for name, dotted in _sdk_import_bindings(tree)})
-
-
-def _resolved(dotted: str, aliases: Mapping[str, str]) -> str:
-    root, _, rest = dotted.partition(".")
-    base: Final = aliases.get(root, root)
-    return f"{base}.{rest}" if rest else base
-
-
-def _patch_targets(call: ast.Call, aliases: Mapping[str, str]) -> Iterator[str]:
-    """What a patch installer is replacing: the dotted string it names, or the
-    attribute chain handed to `patch.object` / `patch.dict`, resolved through the
-    module's imports so a locally bound SDK object reads as its full path."""
-    for first in call.args[:1]:
-        if isinstance(first, ast.Constant) and isinstance(first.value, str):
-            yield first.value
-        elif dotted := _dotted_name(first):
-            yield _resolved(dotted, aliases)
-
-
-def iter_internal_patch_violations(path: Path, tree: ast.Module) -> Iterator[Violation]:
-    aliases: Final = _sdk_aliases(tree)
-    for node in ast.walk(tree):
-        if not (isinstance(node, ast.Call) and _is_patch_installer(_dotted_name(node.func))):
-            continue
-        for target in _patch_targets(node, aliases):
-            if _is_sdk_internal(target):
-                yield Violation(
-                    path,
-                    node.lineno,
-                    "TQ008",
-                    f"patches `{target}`, an SDK internal, so the test is pinned to how the code is "
-                    "wired rather than what it does; fake the HTTP boundary (respx / MockTransport) "
-                    f"or inject the collaborator (suppress: `# {SUPPRESSION_TOKEN}: <reason>`)",
-                )
-
-
 def _environ_keys(node: ast.AST) -> Iterator[str]:
     for inner in ast.walk(node):
         if isinstance(inner, ast.Call) and _dotted_name(inner.func) in ENVIRON_READERS:
@@ -784,7 +719,6 @@ def check_file(path: Path) -> tuple[Violation, ...]:
             *iter_global_mutation_violations(path, tree),
             *iter_credential_skip_violations(path, tree),
             *iter_conftest_inventory_violations(path, tree),
-            *iter_internal_patch_violations(path, tree),
             *iter_child_interpreter_violations(path, tree),
         )
         if violation.line not in skip

test-quality-budget.json

@@ -20,9 +20,6 @@
   "TQ007": {
     "limit": 117
   },
-  "TQ008": {
-    "limit": 10993
-  },
   "TQ009": {
     "limit": 59
   }

tests/AGENTS.md

@@ -22,7 +22,7 @@ Rates in the test, expected computed by hand, one call, `response.text` in the a
 Assert the whole value. Iterating `expected_body.items()` (`test_responses_api_request_body.py`) cannot
 see an extra key; that is the shape of `stream_options.include_usage` (#19777, #28553)
 
-The linter catches no-assert, mock-echo, credential skips and patched internals. It cannot see an assert
+The linter catches no-assert, mock-echo and credential skips. It cannot see an assert
 behind an `if` (a poll that ends in `pytest.fail` is fine), `except Exception` around the call
 (`test_router.py`: `except Exception as e: print(f"FAILED TEST")`), or blanket `--reruns`
 

tests/test_litellm/test_check_test_quality.py

@@ -187,7 +187,7 @@ def test_mock_echo_is_flagged(tmp_path):
         "        run()\n"
         "    mock_completion.assert_called_once()\n"
     )
-    assert _codes(tmp_path, source) == ["TQ002", "TQ008"]
+    assert _codes(tmp_path, source) == ["TQ002"]
 
 
 def test_call_args_inspection_is_mock_echo(tmp_path):
@@ -200,7 +200,7 @@ def test_call_args_inspection_is_mock_echo(tmp_path):
         "        run()\n"
         "    assert mock_completion.call_args[1]['model'] == 'gpt-4o'\n"
     )
-    assert _codes(tmp_path, source) == ["TQ002", "TQ008"]
+    assert _codes(tmp_path, source) == ["TQ002"]
 
 
 def test_patch_decorator_counts_as_installing_a_patch(tmp_path):
@@ -213,7 +213,7 @@ def test_patch_decorator_counts_as_installing_a_patch(tmp_path):
         "    run()\n"
         "    mock_completion.assert_called_once()\n"
     )
-    assert _codes(tmp_path, source) == ["TQ002", "TQ008"]
+    assert _codes(tmp_path, source) == ["TQ002"]
 
 
 def test_patching_but_asserting_the_output_is_not_mock_echo(tmp_path):
@@ -227,7 +227,7 @@ def test_patching_but_asserting_the_output_is_not_mock_echo(tmp_path):
         "    mock_completion.assert_called_once()\n"
         "    assert result.choices[0].message.content == 'pong'\n"
     )
-    assert _codes(tmp_path, source) == ["TQ008"]
+    assert _codes(tmp_path, source) == []
 
 
 def test_asserting_without_patching_is_not_mock_echo(tmp_path):
@@ -244,7 +244,7 @@ def test_a_test_with_no_assertions_is_tq001_not_tq002(tmp_path):
         "    with patch('litellm.completion'):\n"
         "        run()\n"
     )
-    assert _codes(tmp_path, source) == ["TQ001", "TQ008"]
+    assert _codes(tmp_path, source) == ["TQ001"]
 
 
 def test_sys_path_insert_is_flagged(tmp_path):
@@ -554,133 +554,6 @@ def test_a_loop_storing_under_a_key_that_is_not_the_loop_variable_is_not_an_inve
     assert [v.code for v in checker.check_file(_written(tmp_path, source))] == []
 
 
-def test_patching_an_sdk_function_by_string_is_flagged(tmp_path):
-    source = 'from unittest.mock import patch\n\n\n@patch("litellm.completion")\ndef test_x(m):\n    assert m\n'
-    assert "TQ008" in _codes(tmp_path, source)
-
-
-def test_patching_a_deep_sdk_path_is_flagged(tmp_path):
-    source = (
-        "from unittest.mock import patch\n\n\n"
-        "def test_x():\n"
-        '    with patch("litellm.llms.openai.chat.handler.OpenAIChatCompletion.completion"):\n'
-        "        assert True\n"
-    )
-    assert "TQ008" in _codes(tmp_path, source)
-
-
-def test_patch_object_rooted_at_the_sdk_is_flagged(tmp_path):
-    source = (
-        "import litellm\nfrom unittest.mock import patch\n\n\n"
-        "def test_x():\n"
-        '    with patch.object(litellm, "api_key", "x"):\n'
-        "        assert True\n"
-    )
-    assert "TQ008" in _codes(tmp_path, source)
-
-
-def test_patch_object_on_a_from_imported_sdk_module_is_flagged(tmp_path):
-    source = (
-        "from litellm.llms.openai.chat import handler\nfrom unittest.mock import patch\n\n\n"
-        "def test_x():\n"
-        '    with patch.object(handler.OpenAIChatCompletion, "completion"):\n'
-        "        assert True\n"
-    )
-    assert "TQ008" in _codes(tmp_path, source)
-
-
-def test_patch_object_on_an_aliased_sdk_module_is_flagged(tmp_path):
-    source = (
-        "import litellm.llms.openai.chat.handler as oai\nfrom unittest.mock import patch\n\n\n"
-        "def test_x():\n"
-        '    with patch.object(oai.OpenAIChatCompletion, "completion"):\n'
-        "        assert True\n"
-    )
-    assert "TQ008" in _codes(tmp_path, source)
-
-
-def test_patch_object_on_a_renamed_sdk_symbol_is_flagged(tmp_path):
-    source = (
-        "from litellm.utils import get_llm_provider as glp\nfrom unittest.mock import patch\n\n\n"
-        "def test_x():\n"
-        '    with patch.object(glp, "__wrapped__"):\n'
-        "        assert True\n"
-    )
-    assert "TQ008" in _codes(tmp_path, source)
-
-
-def test_the_reported_target_is_the_resolved_sdk_path(tmp_path):
-    source = (
-        "from litellm.llms.openai.chat import handler\nfrom unittest.mock import patch\n\n\n"
-        "def test_x():\n"
-        '    with patch.object(handler.OpenAIChatCompletion, "completion"):\n'
-        "        assert True\n"
-    )
-    reported = [v.message for v in checker.check_file(_written(tmp_path, source)) if v.code == "TQ008"]
-    assert reported
-    assert "litellm.llms.openai.chat.handler.OpenAIChatCompletion" in reported[0]
-
-
-def test_patch_object_on_a_from_imported_third_party_is_not_flagged(tmp_path):
-    source = (
-        "from openai import OpenAI\nfrom unittest.mock import patch\n\n\n"
-        "def test_x():\n"
-        '    with patch.object(OpenAI, "chat"):\n'
-        "        assert True\n"
-    )
-    assert "TQ008" not in _codes(tmp_path, source)
-
-
-def test_a_local_name_with_no_sdk_import_behind_it_is_not_flagged(tmp_path):
-    source = (
-        "from unittest.mock import patch\n\n\n"
-        "def test_x(handler):\n"
-        '    with patch.object(handler, "completion"):\n'
-        "        assert True\n"
-    )
-    assert "TQ008" not in _codes(tmp_path, source)
-
-
-def test_mocking_a_third_party_client_is_not_flagged(tmp_path):
-    source = (
-        "from unittest.mock import patch\n\n\n"
-        "def test_x():\n"
-        '    with patch("openai.OpenAI.chat"):\n'
-        "        assert True\n"
-    )
-    assert "TQ008" not in _codes(tmp_path, source)
-
-
-def test_mocking_the_http_transport_is_not_flagged(tmp_path):
-    source = (
-        "from unittest.mock import patch\n\n\n"
-        "def test_x():\n"
-        '    with patch("httpx.AsyncClient.send"):\n'
-        "        assert True\n"
-    )
-    assert "TQ008" not in _codes(tmp_path, source)
-
-
-def test_a_name_merely_starting_with_litellm_is_not_the_sdk(tmp_path):
-    source = (
-        "from unittest.mock import patch\n\n\n"
-        "def test_x():\n"
-        '    with patch("litellm_enterprise.thing.go"):\n'
-        "        assert True\n"
-    )
-    assert "TQ008" not in _codes(tmp_path, source)
-
-
-def test_an_sdk_patch_can_be_suppressed(tmp_path):
-    source = (
-        "from unittest.mock import patch\n\n\n"
-        "def test_x():\n"
-        '    with patch("litellm.completion"):  # test-quality-ok: pinning the router seam\n'
-        "        assert True\n"
-    )
-    assert "TQ008" not in _codes(tmp_path, source)
-
-
 _FANS_OUT = checker._worker_count(checker.PARALLEL_MIN_PATHS) > 1
 _SERIAL_ONLY = "one usable core, so scan_paths stays serial and there is no fan-out to compare"
 

tests/test_litellm/test_test_quality_gate.py

@@ -137,7 +137,7 @@ def test_the_shipped_budget_covers_every_rule_the_checker_can_emit():
 
     budget = json.loads((_REPO_ROOT / "test-quality-budget.json").read_text())
     assert set(budget) == {
-        "TQ001", "TQ002", "TQ003", "TQ004", "TQ005", "TQ006", "TQ007", "TQ008", "TQ009"
+        "TQ001", "TQ002", "TQ003", "TQ004", "TQ005", "TQ006", "TQ007", "TQ009"
     }
     assert all(spec["limit"] >= 0 for spec in budget.values())
 

tests/unit/conftest.py

@@ -0,0 +1,23 @@
+from collections.abc import Iterator
+from typing import Final
+
+import pytest
+from pytest_socket import enable_socket, socket_allow_hosts
+
+LOOPBACK_HOSTS: Final = ["127.0.0.1", "::1"]
+
+
+def _allow_loopback_only() -> None:
+    socket_allow_hosts(LOOPBACK_HOSTS, allow_unix_socket=True)
+
+
+@pytest.fixture(autouse=True, scope="session")
+def block_external_sockets() -> Iterator[None]:
+    _allow_loopback_only()
+    yield
+    enable_socket()
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_runtest_setup() -> None:
+    _allow_loopback_only()

uv.lock

@@ -4702,6 +4702,7 @@ dev = [
     { name = "pytest-postgresql" },
     { name = "pytest-recording" },
     { name = "pytest-rerunfailures" },
+    { name = "pytest-socket" },
     { name = "pytest-timeout" },
     { name = "pytest-xdist" },
     { name = "reportlab" },
@@ -4902,6 +4903,7 @@ dev = [
     { name = "pytest-postgresql", specifier = "==7.0.2" },
     { name = "pytest-recording", specifier = "==0.13.4" },
     { name = "pytest-rerunfailures", specifier = "==15.1" },
+    { name = "pytest-socket", specifier = "==0.8.1" },
     { name = "pytest-timeout", specifier = "==2.4.0" },
     { name = "pytest-xdist", specifier = "==3.8.0" },
     { name = "reportlab", specifier = "==5.0.1" },
@@ -8087,6 +8089,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/7c/ff/3266c8a73b9b93c4b14160a7e2b31d1e1088e28ed29f4c2d93ae34093bfd/pytest_retry-1.7.0-py3-none-any.whl", hash = "sha256:a2dac85b79a4e2375943f1429479c65beb6c69553e7dae6b8332be47a60954f4", size = 13775, upload-time = "2025-01-19T01:56:11.199Z" },
 ]
 
+[[package]]
+name = "pytest-socket"
+version = "0.8.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ba/ce/4ef7b049852c95a8727b4a7e6496f762df1ac0b47bc0320d10293f5e95ec/pytest_socket-0.8.1.tar.gz", hash = "sha256:2f57787914ad2e1308d09ce141b95c3e55741fbb4fb7b7556593a6b063e0c9c7", size = 17313, upload-time = "2026-08-19T15:16:25.653Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/87/ef/ab507f117b3d19b54e3c9c632a99c28c3b284562ec6e02e274581d530d92/pytest_socket-0.8.1-py3-none-any.whl", hash = "sha256:f9846bed1dcd96eed459e5e14795bbaf96715cf4e827891fe70773817ecb8ed4", size = 8751, upload-time = "2026-08-19T15:16:24.426Z" },
+]
+
 [[package]]
 name = "pytest-timeout"
 version = "2.4.0"