diff --git a/.circleci/config.yml b/.circleci/config.yml index 2dcedbfac4a..474d0af4629 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3142,6 +3142,33 @@ jobs: - store_artifacts: path: test-results + unit: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - setup_litellm_test_deps + - run: + name: Generate Prisma client + command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + - run: + name: Run unit tests + command: | + mkdir -p test-results/unit + mapfile -t files < <(find tests/unit -name 'test_*.py' | sort) + if [ "${#files[@]}" -eq 0 ]; then echo "tests/unit holds no test_*.py files; nothing to run"; exit 0; fi + set +e + LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short --junitxml=test-results/unit/junit.xml + status=$? + set -e + if [ "$status" -eq 5 ]; then echo "pytest collected no tests from tests/unit; passing"; exit 0; fi + exit "$status" + - store_test_results: + path: test-results + - store_artifacts: + path: test-results + workflows: migration_startup: when: << pipeline.parameters.run_migration_tests >> @@ -3190,6 +3217,8 @@ workflows: only: - main - /litellm_.*/ + - unit: + filters: *main_branches - provider_replay_harness - base_sdk_install: filters: *main_branches diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index d4e9a65e7c0..f4d4fb54ba6 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -165,33 +165,41 @@ jobs: DIST: ${{ inputs.dist }} COVERAGE_CORE: sysmon run: | - if [ "${WORKERS}" = "0" ]; then - uv run --no-sync pytest ${TEST_PATH:?} \ - --tb=short -vv \ - --maxfail="${MAX_FAILURES}" \ - --reruns "${RERUNS}" \ - --reruns-delay 1 \ - --timeout="${TEST_TIMEOUT_SECONDS}" \ - --rerun-except "from pytest-timeout" \ - --durations=20 \ - --cov=./litellm --cov=./enterprise/litellm_enterprise \ - --cov-report=xml:coverage.xml \ - --cov-config=pyproject.toml - else - uv run --no-sync pytest ${TEST_PATH:?} \ - --tb=short -vv \ - --maxfail="${MAX_FAILURES}" \ - -n "${WORKERS}" \ - --reruns "${RERUNS}" \ - --reruns-delay 1 \ - --timeout="${TEST_TIMEOUT_SECONDS}" \ - --rerun-except "from pytest-timeout" \ - --dist="${DIST}" \ - --durations=20 \ - --cov=./litellm --cov=./enterprise/litellm_enterprise \ - --cov-report=xml:coverage.xml \ - --cov-config=pyproject.toml + found_path=false + for path in ${TEST_PATH}; do + if [ -e "${path%%::*}" ]; then + found_path=true + break + fi + done + if [ "$found_path" = false ]; then + echo "No path in TEST_PATH exists (${TEST_PATH}); nothing to run" + exit 0 fi + xdist_args=() + if [ "${WORKERS}" != "0" ]; then + xdist_args=(-n "${WORKERS}" --dist="${DIST}") + fi + set +e + uv run --no-sync pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + "${xdist_args[@]}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --timeout="${TEST_TIMEOUT_SECONDS}" \ + --rerun-except "from pytest-timeout" \ + --durations=20 \ + --cov=./litellm --cov=./enterprise/litellm_enterprise \ + --cov-report=xml:coverage.xml \ + --cov-config=pyproject.toml + status=$? + set -e + if [ "$status" -eq 5 ]; then + echo "pytest collected no tests from ${TEST_PATH}; passing" + exit 0 + fi + exit "$status" - name: Save coverage report if: always() && steps.changes.outputs.decision != 'skip' diff --git a/pyproject.toml b/pyproject.toml index 821f885dbfc..3b17a397f3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,6 +196,7 @@ dev = [ "mypy==1.20.1", "keyring==25.7.0", "pytest==9.0.3", + "pytest-socket==0.8.1", "tomli==2.4.1; python_version < '3.11'", "pytest-mock==3.15.1", "pytest-asyncio==1.3.0", diff --git a/scripts/check_test_quality.py b/scripts/check_test_quality.py index 1ef4aed8675..dddc9d61982 100644 --- a/scripts/check_test_quality.py +++ b/scripts/check_test_quality.py @@ -48,10 +48,6 @@ TQ006 A `pytest.skip` reached only when a credential-shaped environment variab deliberate branch. The gate follows one local or module-level binding, which is the `key = os.getenv(...)` then `if not key: pytest.skip(...)` shape most of these use. -TQ008 A `patch(...)` whose target is a `litellm.` internal. Patching the SDK's own - functions pins the test to the current wiring instead of the behaviour, and it - is the idiom the suite reaches for instead of faking the HTTP boundary. Mocking - a third-party client, a transport, or anything outside `litellm.` is untouched. TQ007 A module global that a conftest saves before every test and restores after it. The save/restore list is a hand-maintained inventory of the leaks the suite already knows about, so it is allowed to shrink and never to grow: a new entry @@ -483,67 +479,6 @@ def iter_global_mutation_violations(path: Path, tree: ast.Module) -> Iterator[Vi ) -def _is_sdk_internal(dotted: str) -> bool: - return dotted == SDK_MODULE or dotted.startswith(f"{SDK_MODULE}.") - - -def _sdk_import_bindings(tree: ast.Module) -> Iterator[tuple[str, str]]: - """(local name, dotted path) for every import that binds something under `litellm`.""" - for node in ast.walk(tree): - if isinstance(node, ast.Import): - yield from ( - (alias.asname, alias.name) if alias.asname else (root, root) - for alias in node.names - if _is_sdk_internal(alias.name) - for root in (alias.name.partition(".")[0],) - ) - elif isinstance(node, ast.ImportFrom) and node.module and _is_sdk_internal(node.module): - yield from ((alias.asname or alias.name, f"{node.module}.{alias.name}") for alias in node.names) - - -def _sdk_aliases(tree: ast.Module) -> Mapping[str, str]: - """Local names bound to something under `litellm`, mapped to the path they stand for. - - `from litellm.llms.openai.chat import handler` then `patch.object(handler.X, ...)` - reaches the same internal as the dotted string form and has to read the same way. - """ - return MappingProxyType({name: dotted for name, dotted in _sdk_import_bindings(tree)}) - - -def _resolved(dotted: str, aliases: Mapping[str, str]) -> str: - root, _, rest = dotted.partition(".") - base: Final = aliases.get(root, root) - return f"{base}.{rest}" if rest else base - - -def _patch_targets(call: ast.Call, aliases: Mapping[str, str]) -> Iterator[str]: - """What a patch installer is replacing: the dotted string it names, or the - attribute chain handed to `patch.object` / `patch.dict`, resolved through the - module's imports so a locally bound SDK object reads as its full path.""" - for first in call.args[:1]: - if isinstance(first, ast.Constant) and isinstance(first.value, str): - yield first.value - elif dotted := _dotted_name(first): - yield _resolved(dotted, aliases) - - -def iter_internal_patch_violations(path: Path, tree: ast.Module) -> Iterator[Violation]: - aliases: Final = _sdk_aliases(tree) - for node in ast.walk(tree): - if not (isinstance(node, ast.Call) and _is_patch_installer(_dotted_name(node.func))): - continue - for target in _patch_targets(node, aliases): - if _is_sdk_internal(target): - yield Violation( - path, - node.lineno, - "TQ008", - f"patches `{target}`, an SDK internal, so the test is pinned to how the code is " - "wired rather than what it does; fake the HTTP boundary (respx / MockTransport) " - f"or inject the collaborator (suppress: `# {SUPPRESSION_TOKEN}: `)", - ) - - def _environ_keys(node: ast.AST) -> Iterator[str]: for inner in ast.walk(node): if isinstance(inner, ast.Call) and _dotted_name(inner.func) in ENVIRON_READERS: @@ -784,7 +719,6 @@ def check_file(path: Path) -> tuple[Violation, ...]: *iter_global_mutation_violations(path, tree), *iter_credential_skip_violations(path, tree), *iter_conftest_inventory_violations(path, tree), - *iter_internal_patch_violations(path, tree), *iter_child_interpreter_violations(path, tree), ) if violation.line not in skip diff --git a/test-quality-budget.json b/test-quality-budget.json index ae4ea4d31be..6f3dde8461b 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -20,9 +20,6 @@ "TQ007": { "limit": 117 }, - "TQ008": { - "limit": 10993 - }, "TQ009": { "limit": 59 } diff --git a/tests/AGENTS.md b/tests/AGENTS.md index ad2b8d95eaf..13b4789003f 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -22,7 +22,7 @@ Rates in the test, expected computed by hand, one call, `response.text` in the a Assert the whole value. Iterating `expected_body.items()` (`test_responses_api_request_body.py`) cannot see an extra key; that is the shape of `stream_options.include_usage` (#19777, #28553) -The linter catches no-assert, mock-echo, credential skips and patched internals. It cannot see an assert +The linter catches no-assert, mock-echo and credential skips. It cannot see an assert behind an `if` (a poll that ends in `pytest.fail` is fine), `except Exception` around the call (`test_router.py`: `except Exception as e: print(f"FAILED TEST")`), or blanket `--reruns` diff --git a/tests/test_litellm/test_check_test_quality.py b/tests/test_litellm/test_check_test_quality.py index bfe503e74d1..05c25fb19fb 100644 --- a/tests/test_litellm/test_check_test_quality.py +++ b/tests/test_litellm/test_check_test_quality.py @@ -187,7 +187,7 @@ def test_mock_echo_is_flagged(tmp_path): " run()\n" " mock_completion.assert_called_once()\n" ) - assert _codes(tmp_path, source) == ["TQ002", "TQ008"] + assert _codes(tmp_path, source) == ["TQ002"] def test_call_args_inspection_is_mock_echo(tmp_path): @@ -200,7 +200,7 @@ def test_call_args_inspection_is_mock_echo(tmp_path): " run()\n" " assert mock_completion.call_args[1]['model'] == 'gpt-4o'\n" ) - assert _codes(tmp_path, source) == ["TQ002", "TQ008"] + assert _codes(tmp_path, source) == ["TQ002"] def test_patch_decorator_counts_as_installing_a_patch(tmp_path): @@ -213,7 +213,7 @@ def test_patch_decorator_counts_as_installing_a_patch(tmp_path): " run()\n" " mock_completion.assert_called_once()\n" ) - assert _codes(tmp_path, source) == ["TQ002", "TQ008"] + assert _codes(tmp_path, source) == ["TQ002"] def test_patching_but_asserting_the_output_is_not_mock_echo(tmp_path): @@ -227,7 +227,7 @@ def test_patching_but_asserting_the_output_is_not_mock_echo(tmp_path): " mock_completion.assert_called_once()\n" " assert result.choices[0].message.content == 'pong'\n" ) - assert _codes(tmp_path, source) == ["TQ008"] + assert _codes(tmp_path, source) == [] def test_asserting_without_patching_is_not_mock_echo(tmp_path): @@ -244,7 +244,7 @@ def test_a_test_with_no_assertions_is_tq001_not_tq002(tmp_path): " with patch('litellm.completion'):\n" " run()\n" ) - assert _codes(tmp_path, source) == ["TQ001", "TQ008"] + assert _codes(tmp_path, source) == ["TQ001"] def test_sys_path_insert_is_flagged(tmp_path): @@ -554,133 +554,6 @@ def test_a_loop_storing_under_a_key_that_is_not_the_loop_variable_is_not_an_inve assert [v.code for v in checker.check_file(_written(tmp_path, source))] == [] -def test_patching_an_sdk_function_by_string_is_flagged(tmp_path): - source = 'from unittest.mock import patch\n\n\n@patch("litellm.completion")\ndef test_x(m):\n assert m\n' - assert "TQ008" in _codes(tmp_path, source) - - -def test_patching_a_deep_sdk_path_is_flagged(tmp_path): - source = ( - "from unittest.mock import patch\n\n\n" - "def test_x():\n" - ' with patch("litellm.llms.openai.chat.handler.OpenAIChatCompletion.completion"):\n' - " assert True\n" - ) - assert "TQ008" in _codes(tmp_path, source) - - -def test_patch_object_rooted_at_the_sdk_is_flagged(tmp_path): - source = ( - "import litellm\nfrom unittest.mock import patch\n\n\n" - "def test_x():\n" - ' with patch.object(litellm, "api_key", "x"):\n' - " assert True\n" - ) - assert "TQ008" in _codes(tmp_path, source) - - -def test_patch_object_on_a_from_imported_sdk_module_is_flagged(tmp_path): - source = ( - "from litellm.llms.openai.chat import handler\nfrom unittest.mock import patch\n\n\n" - "def test_x():\n" - ' with patch.object(handler.OpenAIChatCompletion, "completion"):\n' - " assert True\n" - ) - assert "TQ008" in _codes(tmp_path, source) - - -def test_patch_object_on_an_aliased_sdk_module_is_flagged(tmp_path): - source = ( - "import litellm.llms.openai.chat.handler as oai\nfrom unittest.mock import patch\n\n\n" - "def test_x():\n" - ' with patch.object(oai.OpenAIChatCompletion, "completion"):\n' - " assert True\n" - ) - assert "TQ008" in _codes(tmp_path, source) - - -def test_patch_object_on_a_renamed_sdk_symbol_is_flagged(tmp_path): - source = ( - "from litellm.utils import get_llm_provider as glp\nfrom unittest.mock import patch\n\n\n" - "def test_x():\n" - ' with patch.object(glp, "__wrapped__"):\n' - " assert True\n" - ) - assert "TQ008" in _codes(tmp_path, source) - - -def test_the_reported_target_is_the_resolved_sdk_path(tmp_path): - source = ( - "from litellm.llms.openai.chat import handler\nfrom unittest.mock import patch\n\n\n" - "def test_x():\n" - ' with patch.object(handler.OpenAIChatCompletion, "completion"):\n' - " assert True\n" - ) - reported = [v.message for v in checker.check_file(_written(tmp_path, source)) if v.code == "TQ008"] - assert reported - assert "litellm.llms.openai.chat.handler.OpenAIChatCompletion" in reported[0] - - -def test_patch_object_on_a_from_imported_third_party_is_not_flagged(tmp_path): - source = ( - "from openai import OpenAI\nfrom unittest.mock import patch\n\n\n" - "def test_x():\n" - ' with patch.object(OpenAI, "chat"):\n' - " assert True\n" - ) - assert "TQ008" not in _codes(tmp_path, source) - - -def test_a_local_name_with_no_sdk_import_behind_it_is_not_flagged(tmp_path): - source = ( - "from unittest.mock import patch\n\n\n" - "def test_x(handler):\n" - ' with patch.object(handler, "completion"):\n' - " assert True\n" - ) - assert "TQ008" not in _codes(tmp_path, source) - - -def test_mocking_a_third_party_client_is_not_flagged(tmp_path): - source = ( - "from unittest.mock import patch\n\n\n" - "def test_x():\n" - ' with patch("openai.OpenAI.chat"):\n' - " assert True\n" - ) - assert "TQ008" not in _codes(tmp_path, source) - - -def test_mocking_the_http_transport_is_not_flagged(tmp_path): - source = ( - "from unittest.mock import patch\n\n\n" - "def test_x():\n" - ' with patch("httpx.AsyncClient.send"):\n' - " assert True\n" - ) - assert "TQ008" not in _codes(tmp_path, source) - - -def test_a_name_merely_starting_with_litellm_is_not_the_sdk(tmp_path): - source = ( - "from unittest.mock import patch\n\n\n" - "def test_x():\n" - ' with patch("litellm_enterprise.thing.go"):\n' - " assert True\n" - ) - assert "TQ008" not in _codes(tmp_path, source) - - -def test_an_sdk_patch_can_be_suppressed(tmp_path): - source = ( - "from unittest.mock import patch\n\n\n" - "def test_x():\n" - ' with patch("litellm.completion"): # test-quality-ok: pinning the router seam\n' - " assert True\n" - ) - assert "TQ008" not in _codes(tmp_path, source) - - _FANS_OUT = checker._worker_count(checker.PARALLEL_MIN_PATHS) > 1 _SERIAL_ONLY = "one usable core, so scan_paths stays serial and there is no fan-out to compare" diff --git a/tests/test_litellm/test_test_quality_gate.py b/tests/test_litellm/test_test_quality_gate.py index cde33787c6c..873e7b9b336 100644 --- a/tests/test_litellm/test_test_quality_gate.py +++ b/tests/test_litellm/test_test_quality_gate.py @@ -137,7 +137,7 @@ def test_the_shipped_budget_covers_every_rule_the_checker_can_emit(): budget = json.loads((_REPO_ROOT / "test-quality-budget.json").read_text()) assert set(budget) == { - "TQ001", "TQ002", "TQ003", "TQ004", "TQ005", "TQ006", "TQ007", "TQ008", "TQ009" + "TQ001", "TQ002", "TQ003", "TQ004", "TQ005", "TQ006", "TQ007", "TQ009" } assert all(spec["limit"] >= 0 for spec in budget.values()) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 00000000000..3bdab1d231a --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,23 @@ +from collections.abc import Iterator +from typing import Final + +import pytest +from pytest_socket import enable_socket, socket_allow_hosts + +LOOPBACK_HOSTS: Final = ["127.0.0.1", "::1"] + + +def _allow_loopback_only() -> None: + socket_allow_hosts(LOOPBACK_HOSTS, allow_unix_socket=True) + + +@pytest.fixture(autouse=True, scope="session") +def block_external_sockets() -> Iterator[None]: + _allow_loopback_only() + yield + enable_socket() + + +@pytest.hookimpl(trylast=True) +def pytest_runtest_setup() -> None: + _allow_loopback_only() diff --git a/uv.lock b/uv.lock index f1a58500a61..dfba6bfc258 100644 --- a/uv.lock +++ b/uv.lock @@ -4702,6 +4702,7 @@ dev = [ { name = "pytest-postgresql" }, { name = "pytest-recording" }, { name = "pytest-rerunfailures" }, + { name = "pytest-socket" }, { name = "pytest-timeout" }, { name = "pytest-xdist" }, { name = "reportlab" }, @@ -4902,6 +4903,7 @@ dev = [ { name = "pytest-postgresql", specifier = "==7.0.2" }, { name = "pytest-recording", specifier = "==0.13.4" }, { name = "pytest-rerunfailures", specifier = "==15.1" }, + { name = "pytest-socket", specifier = "==0.8.1" }, { name = "pytest-timeout", specifier = "==2.4.0" }, { name = "pytest-xdist", specifier = "==3.8.0" }, { name = "reportlab", specifier = "==5.0.1" }, @@ -8087,6 +8089,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/ff/3266c8a73b9b93c4b14160a7e2b31d1e1088e28ed29f4c2d93ae34093bfd/pytest_retry-1.7.0-py3-none-any.whl", hash = "sha256:a2dac85b79a4e2375943f1429479c65beb6c69553e7dae6b8332be47a60954f4", size = 13775, upload-time = "2025-01-19T01:56:11.199Z" }, ] +[[package]] +name = "pytest-socket" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ba/ce/4ef7b049852c95a8727b4a7e6496f762df1ac0b47bc0320d10293f5e95ec/pytest_socket-0.8.1.tar.gz", hash = "sha256:2f57787914ad2e1308d09ce141b95c3e55741fbb4fb7b7556593a6b063e0c9c7", size = 17313, upload-time = "2026-08-19T15:16:25.653Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/ef/ab507f117b3d19b54e3c9c632a99c28c3b284562ec6e02e274581d530d92/pytest_socket-0.8.1-py3-none-any.whl", hash = "sha256:f9846bed1dcd96eed459e5e14795bbaf96715cf4e827891fe70773817ecb8ed4", size = 8751, upload-time = "2026-08-19T15:16:24.426Z" }, +] + [[package]] name = "pytest-timeout" version = "2.4.0"