diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1fe0c602036..db46114715d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -65,7 +65,7 @@ After: the same request comes back with real token counts, so the dashboard show **Please complete all items before asking a LiteLLM maintainer to review your PR** - [ ] I have added meaningful tests -- [ ] The handful of test files covering my change pass locally, e.g. `uv run pytest tests/test_litellm/.py -v`. Leave the suites (`make test-unit-*`, `make test-unit`) to CI: it finishes in ~15 minutes where a laptop takes an hour or more +- [ ] The handful of test files covering my change pass locally, e.g. `uv run pytest tests/unit/.py -v`. Leave the suites (`make test-unit-*`, `make test-unit`) to CI: it finishes in ~15 minutes where a laptop takes an hour or more - [ ] My PR passes all required CI/CD checks (e.g., lint, schema.d.ts sync check, etc.) - [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem - [ ] I have received a Greptile **Confidence Score of at least 4/5** before requesting a maintainer review (Greptile reviews automatically once the PR is opened; only comment `@greptileai` to re-request a review after pushing changes) diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 808bb2afd08..2d399cca3a4 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -14,7 +14,6 @@ on: - "litellm/ocr/**" - "litellm/llms/base_llm/ocr/**" - "litellm/llms/custom_httpx/llm_http_handler.py" - - "tests/test_litellm/ocr/**" - "tests/test_litellm/conftest.py" - "Makefile" - ".cargo/**" @@ -42,7 +41,6 @@ on: - "litellm/ocr/**" - "litellm/llms/base_llm/ocr/**" - "litellm/llms/custom_httpx/llm_http_handler.py" - - "tests/test_litellm/ocr/**" - "tests/test_litellm/conftest.py" - "Makefile" - ".cargo/**" diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 2212b276b0d..f55e186e3b2 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -88,7 +88,7 @@ jobs: - shard: Vertex AI artifact-name: llm-vertex-ai - test-path: "tests/test_litellm/llms/vertex_ai" + test-path: "" unit-flag: llm-vertex-ai workers: 1 reruns: 2 @@ -97,7 +97,7 @@ jobs: - shard: All Other Providers artifact-name: llm-other-providers - test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" + test-path: "" unit-flag: llm-other-providers workers: 2 reruns: 2 @@ -107,9 +107,6 @@ jobs: - shard: misc artifact-name: misc test-path: >- - tests/test_litellm/interactions - tests/test_litellm/ocr - tests/test_litellm/passthrough tests/test_litellm/test_*.py unit-flag: misc workers: 2 diff --git a/AGENTS.md b/AGENTS.md index 69e034fbdea..a2dcd24bdd1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,7 +27,7 @@ Never test structure of code only function of it A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent"; assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken -`tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_.py` if you're the first test there). One focused regression test beats many shallow ones +`tests/unit/` mirrors `litellm/` in a parallel path (see `tests/unit/AGENTS.md`). Name tests `test_.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_.py` if you're the first test there). One focused regression test beats many shallow ones End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `AGENTS.md` diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f418752d990..c9e046748e8 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -255,7 +255,7 @@ Conventions to follow when touching this layer: | Column vs. field names | Where a model field differs from its DB column (for example `org_id` maps to the `organization_id` column), the repository translates in both directions rather than relying on Pydantic to guess. | | Array mutations | Adds use Prisma's atomic `push` (`add_member`, `add_admin`, `add_models`) to avoid read-modify-write races. Removals fall back to read-modify-write because Prisma has no atomic array remove. | -To add a new entity, define the model under `litellm/models/`, re-export it from `proxy/_types.py` if existing code imports it from there, and add a repository under `litellm/repositories/` (subclass `BaseRepository` for plain CRUD, or add bespoke methods when the entity needs encryption, archiving, or atomic array updates). Mirror the tests in `tests/test_litellm/repositories/`. +To add a new entity, define the model under `litellm/models/`, re-export it from `proxy/_types.py` if existing code imports it from there, and add a repository under `litellm/repositories/` (subclass `BaseRepository` for plain CRUD, or add bespoke methods when the entity needs encryption, archiving, or atomic array updates). Mirror the tests in `tests/unit/repositories/`. --- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 082b7a8fb3e..a5ad6e97f3d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,7 +14,7 @@ Here are the core requirements for any PR submitted to LiteLLM: - [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing) - [ ] **Ensure your PR passes all checks**: - [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint` - - [ ] [The tests covering your change](#running-unit-tests) pass, e.g. `uv run pytest tests/test_litellm/.py -v`. CI runs the full unit test matrix, so you don't need to run the whole suite locally + - [ ] [The tests covering your change](#running-unit-tests) pass, e.g. `uv run pytest tests/unit/.py -v`. CI runs the full unit test matrix, so you don't need to run the whole suite locally #### UI PRs @@ -72,7 +72,7 @@ make format make lint # Run the tests covering your change (CI runs the full suite) -uv run pytest tests/test_litellm/.py -v +uv run pytest tests/unit/.py -v # Commit your changes (must follow Conventional Commits — see above) git add . @@ -88,7 +88,7 @@ git push origin feature/your-feature ### Where to Add Tests -Add your tests to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm). +Add your tests to the [`tests/unit/` directory](https://github.com/BerriAI/litellm/tree/main/tests/unit). - This directory mirrors the structure of the `litellm/` directory - **Only add mocked tests** - no real LLM API calls in this directory @@ -96,10 +96,10 @@ Add your tests to the [`tests/test_litellm/` directory](https://github.com/Berri ### File Naming Convention -The `tests/test_litellm/` directory follows the same structure as `litellm/`: +The `tests/unit/` directory follows the same structure as `litellm/`: - `litellm/proxy/caching_routes.py` → `tests/test_litellm/proxy/test_caching_routes.py` -- `litellm/utils.py` → `tests/test_litellm/test_utils.py` +- `litellm/utils.py` → `tests/unit/test_utils.py` ### Example Test @@ -125,10 +125,10 @@ def test_your_feature(): Run the tests covering your change: ```bash -uv run pytest tests/test_litellm/test_your_file.py -v +uv run pytest tests/unit/test_your_file.py -v ``` -`tests/test_litellm` holds thousands of tests, so running all of it locally takes a long time. CI runs it as a parallel matrix (`make test-unit-llms`, `make test-unit-proxy-core`, and the other `test-unit-*` targets) on beefier boxes, so if, for whatever reason, you must run the whole suite, it's better to rely on CI to do that. +`tests/unit` holds thousands of tests, so running all of it locally takes a long time. CI runs it as a parallel matrix (`make test-unit-llms`, `make test-unit-proxy-core`, and the other `test-unit-*` targets) on beefier boxes, so if, for whatever reason, you must run the whole suite, it's better to rely on CI to do that. If you're running broader test suites, proxy tests, or anything that touches PostgreSQL-backed fixtures/plugins, install the full local test environment first: diff --git a/Makefile b/Makefile index 311a7daef92..79c18f6fe82 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ help: @echo " make check-circular-imports - Check for circular imports" @echo " make check-import-safety - Check import safety" @echo " make test - Run all tests" - @echo " make test-unit - Run unit tests (tests/test_litellm)" + @echo " make test-unit - Run unit tests (tests/unit and tests/test_litellm)" @echo " make test-unit-llms - Run LLM provider tests (~225 files)" @echo " make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)" @echo " make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)" @@ -310,7 +310,7 @@ test: install-test-deps $(UV_RUN) pytest tests/ test-unit: install-test-deps - $(UV_RUN) pytest tests/test_litellm -x -vv -n 4 + $(UV_RUN) pytest tests/unit tests/test_litellm -x -vv -n 4 # Matrix test targets (matching CI workflow groups) test-unit-llms: install-test-deps @@ -332,7 +332,7 @@ test-unit-core-utils: install-test-deps $(UV_RUN) pytest tests/unit/litellm_core_utils --tb=short -vv -n 2 --durations=20 test-unit-other: install-test-deps - $(UV_RUN) pytest tests/unit/caching tests/unit/responses tests/unit/secret_managers tests/unit/vector_stores tests/unit/a2a_protocol tests/test_litellm/anthropic_interface tests/unit/completion_extras tests/unit/containers tests/unit/enterprise tests/unit/experimental_mcp_client tests/unit/google_genai tests/unit/images tests/unit/interactions tests/test_litellm/interactions tests/test_litellm/passthrough tests/unit/router_strategy tests/unit/router_utils tests/unit/types --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/unit/caching tests/unit/responses tests/unit/secret_managers tests/unit/vector_stores tests/unit/a2a_protocol tests/unit/completion_extras tests/unit/containers tests/unit/enterprise tests/unit/experimental_mcp_client tests/unit/google_genai tests/unit/images tests/unit/interactions tests/unit/router_strategy tests/unit/router_utils tests/unit/types --tb=short -vv -n 4 --durations=20 test-unit-root: install-test-deps $(UV_RUN) pytest tests/unit/test_*.py tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20 diff --git a/litellm/containers/README.md b/litellm/containers/README.md index b54f96b1132..571bcaf415c 100644 --- a/litellm/containers/README.md +++ b/litellm/containers/README.md @@ -213,7 +213,7 @@ Run the container API tests: ```bash cd /Users/ishaanjaffer/github/litellm -python -m pytest tests/test_litellm/containers/ -v +python -m pytest tests/unit/containers/ -v ``` Test via proxy: diff --git a/tests/README.MD b/tests/README.MD index 57275a031f7..6a5da137203 100644 --- a/tests/README.MD +++ b/tests/README.MD @@ -4,6 +4,6 @@ To make it easier to contribute and map what behavior is tested, -we've started mapping the litellm directory in `tests/test_litellm` +we've started mapping the litellm directory in `tests/unit` This folder can only run mock tests. diff --git a/tests/integration/sandbox/test_e2b_sandbox.py b/tests/integration/sandbox/test_e2b_sandbox.py index d1cff2ce178..5adfb99db61 100644 --- a/tests/integration/sandbox/test_e2b_sandbox.py +++ b/tests/integration/sandbox/test_e2b_sandbox.py @@ -2,8 +2,7 @@ e2b code execution sandbox - end-to-end integration tests. These tests make REAL HTTP calls to the e2b API and are skipped automatically -unless E2B_API_KEY is set. Mock-only unit tests live in -tests/test_litellm/sandbox/test_e2b_sandbox.py. +unless E2B_API_KEY is set. Run only these tests: pytest tests/integration/sandbox/test_e2b_sandbox.py -v diff --git a/tests/test_litellm/llms/databricks/databricks_config.template.txt b/tests/llm_translation/databricks_config.template.txt similarity index 100% rename from tests/test_litellm/llms/databricks/databricks_config.template.txt rename to tests/llm_translation/databricks_config.template.txt diff --git a/tests/test_litellm/interactions/base_interactions_test.py b/tests/llm_translation/interactions/base_interactions_test.py similarity index 100% rename from tests/test_litellm/interactions/base_interactions_test.py rename to tests/llm_translation/interactions/base_interactions_test.py diff --git a/tests/test_litellm/interactions/test_gemini_interactions.py b/tests/llm_translation/interactions/test_gemini_interactions.py similarity index 88% rename from tests/test_litellm/interactions/test_gemini_interactions.py rename to tests/llm_translation/interactions/test_gemini_interactions.py index afce77e3ce4..0ab4da952e6 100644 --- a/tests/test_litellm/interactions/test_gemini_interactions.py +++ b/tests/llm_translation/interactions/test_gemini_interactions.py @@ -6,7 +6,7 @@ Inherits from BaseInteractionsTest to run the same test suite against Gemini. import os -from tests.test_litellm.interactions.base_interactions_test import ( +from tests.llm_translation.interactions.base_interactions_test import ( BaseInteractionsTest, ) diff --git a/tests/test_litellm/interactions/test_google_interactions_integration.py b/tests/llm_translation/interactions/test_google_interactions_integration.py similarity index 99% rename from tests/test_litellm/interactions/test_google_interactions_integration.py rename to tests/llm_translation/interactions/test_google_interactions_integration.py index 93429d64789..10e6cf86e6d 100644 --- a/tests/test_litellm/interactions/test_google_interactions_integration.py +++ b/tests/llm_translation/interactions/test_google_interactions_integration.py @@ -5,7 +5,7 @@ Tests the litellm.interactions.create() and related methods against the Google A Per OpenAPI spec: https://ai.google.dev/static/api/interactions.openapi.json -Run with: pytest tests/test_litellm/interactions/test_google_interactions_integration.py -v +Run with: pytest tests/llm_translation/interactions/test_google_interactions_integration.py -v """ import asyncio diff --git a/tests/test_litellm/interactions/test_litellm_responses_bridge.py b/tests/llm_translation/interactions/test_litellm_responses_bridge.py similarity index 91% rename from tests/test_litellm/interactions/test_litellm_responses_bridge.py rename to tests/llm_translation/interactions/test_litellm_responses_bridge.py index 17e7f9fc4ff..ae025ab60b0 100644 --- a/tests/test_litellm/interactions/test_litellm_responses_bridge.py +++ b/tests/llm_translation/interactions/test_litellm_responses_bridge.py @@ -7,7 +7,7 @@ the litellm_responses bridge provider, which calls litellm.responses() internall import os -from tests.test_litellm.interactions.base_interactions_test import ( +from tests.llm_translation.interactions.base_interactions_test import ( BaseInteractionsTest, ) diff --git a/tests/test_litellm/llms/cometapi/chat/test_cometapi_chat_transformation.py b/tests/llm_translation/test_cometapi_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/cometapi/chat/test_cometapi_chat_transformation.py rename to tests/llm_translation/test_cometapi_chat_transformation.py diff --git a/tests/test_litellm/test_compression.py b/tests/llm_translation/test_compression.py similarity index 100% rename from tests/test_litellm/test_compression.py rename to tests/llm_translation/test_compression.py diff --git a/tests/test_litellm/llms/databricks/test_databricks_e2e.py b/tests/llm_translation/test_databricks_e2e.py similarity index 99% rename from tests/test_litellm/llms/databricks/test_databricks_e2e.py rename to tests/llm_translation/test_databricks_e2e.py index 669f9e94639..a979988102e 100644 --- a/tests/test_litellm/llms/databricks/test_databricks_e2e.py +++ b/tests/llm_translation/test_databricks_e2e.py @@ -51,7 +51,7 @@ Setup: Run with: cd /path/to/litellm - python tests/test_litellm/llms/databricks/test_databricks_e2e.py + python tests/llm_translation/test_databricks_e2e.py Config Options: TEST_AUTH_METHOD=oauth # Test OAuth M2M authentication @@ -69,12 +69,12 @@ import pytest # These are E2E tests that require real Databricks credentials pytestmark = pytest.mark.skip( reason="E2E tests require real Databricks credentials. Run directly with: " - "python tests/test_litellm/llms/databricks/test_databricks_e2e.py" + "python tests/llm_translation/test_databricks_e2e.py" ) # Add the litellm package to path sys.path.insert( - 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) ) # Config file path - can be overridden with DATABRICKS_TEST_CONFIG env var diff --git a/tests/test_litellm/llms/openai_like/test_json_providers.py b/tests/llm_translation/test_json_providers.py similarity index 100% rename from tests/test_litellm/llms/openai_like/test_json_providers.py rename to tests/llm_translation/test_json_providers.py diff --git a/tests/test_litellm/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py b/tests/llm_translation/test_mistral_audio_transcription_transformation.py similarity index 100% rename from tests/test_litellm/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py rename to tests/llm_translation/test_mistral_audio_transcription_transformation.py diff --git a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py b/tests/llm_translation/test_ovhcloud_audio_transcription_transformation.py similarity index 100% rename from tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py rename to tests/llm_translation/test_ovhcloud_audio_transcription_transformation.py diff --git a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py b/tests/llm_translation/test_ovhcloud_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py rename to tests/llm_translation/test_ovhcloud_chat_transformation.py diff --git a/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py b/tests/llm_translation/test_vertex_ai_image_generation_transformation.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py rename to tests/llm_translation/test_vertex_ai_image_generation_transformation.py diff --git a/tests/test_litellm/llms/openai_like/test_xiaomi_mimo.py b/tests/llm_translation/test_xiaomi_mimo.py similarity index 100% rename from tests/test_litellm/llms/openai_like/test_xiaomi_mimo.py rename to tests/llm_translation/test_xiaomi_mimo.py diff --git a/tests/local_testing/test_handler_gc_does_not_close_client.py b/tests/local_testing/test_handler_gc_does_not_close_client.py index 63c5694dd89..d6987107fa8 100644 --- a/tests/local_testing/test_handler_gc_does_not_close_client.py +++ b/tests/local_testing/test_handler_gc_does_not_close_client.py @@ -23,10 +23,7 @@ test here may keep the client in a local: that inflates the very refcount under test, and the test then passes on a broken handler. They hold weak references instead, which the refcount does not count. -These live here rather than under ``tests/test_litellm/`` because they need a -real connection pool: a mocked transport goes on yielding chunks after its -client is closed, so the very teardown under test is what a mock cannot -reproduce. The server is a hermetic, credential-free ``ThreadingHTTPServer`` on +The server is a hermetic, credential-free ``ThreadingHTTPServer`` on an ephemeral loopback port, and needs no network access beyond it. Related: https://github.com/BerriAI/litellm/issues/24929 diff --git a/tests/test_litellm/llms/mistral/__init__.py b/tests/test_litellm/llms/mistral/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/test_litellm/llms/openai_like/__init__.py b/tests/test_litellm/llms/openai_like/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/test_litellm/llms/vertex_ai/__init__.py b/tests/test_litellm/llms/vertex_ai/__init__.py deleted file mode 100644 index fc7e977484b..00000000000 --- a/tests/test_litellm/llms/vertex_ai/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Vertex AI tests package.""" diff --git a/tests/test_litellm/log.txt b/tests/test_litellm/log.txt deleted file mode 100644 index 6470b12fedb..00000000000 --- a/tests/test_litellm/log.txt +++ /dev/null @@ -1,2 +0,0 @@ -llms/bedrock/chat/invoke_agent/transformation.py:404: error: Incompatible types in assignment (expression has type "object", variable has type "InvokeAgentModelInvocationOutput | None") [assignment] -llms/bedrock/chat/invoke_agent/transformation.py:405: error: Argument 1 to "get" of "Mapping" has incompatible type "str | InvokeAgentModelInvocationOutput"; expected "str" [typeddict-item] diff --git a/tests/test_litellm/ocr/__init__.py b/tests/test_litellm/ocr/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/test_litellm/passthrough/__init__.py b/tests/test_litellm/passthrough/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/unit/AGENTS.md b/tests/unit/AGENTS.md index 191777f3e83..fc9798ad30e 100644 --- a/tests/unit/AGENTS.md +++ b/tests/unit/AGENTS.md @@ -30,7 +30,7 @@ Green if `send_batched` drops every row. pydantic doubles in 12 of 203 files, fa ## Where it goes `tests/unit/` mirrors `litellm/`, so a changed file selects its tests by path, not a mapping -file. Empty today; new unit tests go here. The examples above live in `tests/test_litellm` +file. New unit tests go here ## Writing it so a human can read it diff --git a/tests/unit/integrations/dotprompt/test_prompt_manager.py b/tests/unit/integrations/dotprompt/test_prompt_manager.py index 51e14b61929..dbd4e4a4c4c 100644 --- a/tests/unit/integrations/dotprompt/test_prompt_manager.py +++ b/tests/unit/integrations/dotprompt/test_prompt_manager.py @@ -22,7 +22,7 @@ def test_prompt_manager_initialization(): # Test with the existing prompts directory prompt_dir = Path( __file__ - ).parent # Current directory when running from tests/test_litellm/prompts + ).parent manager = PromptManager(prompt_directory=str(prompt_dir)) # Should have loaded at least the sample prompts @@ -56,7 +56,7 @@ def test_render_simple_template(): """Test rendering a simple template with variables.""" prompt_dir = Path( __file__ - ).parent # Current directory when running from tests/test_litellm/prompts + ).parent manager = PromptManager(prompt_directory=str(prompt_dir)) # Test sample_prompt rendering @@ -72,7 +72,7 @@ def test_render_chat_prompt(): """Test rendering the chat prompt with conditional content.""" prompt_dir = Path( __file__ - ).parent # Current directory when running from tests/test_litellm/prompts + ).parent manager = PromptManager(prompt_directory=str(prompt_dir)) # Test with system context @@ -98,7 +98,7 @@ def test_render_coding_assistant(): """Test rendering the coding assistant prompt with complex logic.""" prompt_dir = Path( __file__ - ).parent # Current directory when running from tests/test_litellm/prompts + ).parent manager = PromptManager(prompt_directory=str(prompt_dir)) rendered = manager.render( @@ -159,7 +159,7 @@ def test_prompt_not_found(): """Test error handling for non-existent prompts.""" prompt_dir = Path( __file__ - ).parent # Current directory when running from tests/test_litellm/prompts + ).parent manager = PromptManager(prompt_directory=str(prompt_dir)) with pytest.raises(KeyError, match="Prompt 'nonexistent' not found"): @@ -170,7 +170,7 @@ def test_list_prompts(): """Test listing available prompts.""" prompt_dir = Path( __file__ - ).parent # Current directory when running from tests/test_litellm/prompts + ).parent manager = PromptManager(prompt_directory=str(prompt_dir)) prompts = manager.list_prompts() @@ -184,7 +184,7 @@ def test_get_prompt_metadata(): """Test retrieving prompt metadata.""" prompt_dir = Path( __file__ - ).parent # Current directory when running from tests/test_litellm/prompts + ).parent manager = PromptManager(prompt_directory=str(prompt_dir)) metadata = manager.get_prompt_metadata("sample_prompt") @@ -221,7 +221,7 @@ def test_add_prompt_programmatically(): """Test adding prompts programmatically.""" prompt_dir = Path( __file__ - ).parent # Current directory when running from tests/test_litellm/prompts + ).parent manager = PromptManager(prompt_directory=str(prompt_dir)) initial_count = len(manager.prompts) diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index c3478c0d5eb..47c4576f91f 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -1,5 +1,6 @@ """Test health check helper functions""" +import socket import struct import zlib from types import MappingProxyType @@ -214,24 +215,26 @@ async def test_ahealth_check_failure_masks_raw_request_headers(): This tests the fix for the security vulnerability where Authorization headers were being exposed in health check error responses. """ - # Use a model configuration that will fail (invalid endpoint) test_api_key = "dapi-test-key-1234567890abcdef" test_headers = { "Authorization": f"Bearer {test_api_key}", "Content-Type": "application/json", } - response = await ahealth_check( - model_params={ - "model": "databricks/dbrx-instruct", - "api_base": "https://invalid-endpoint-that-will-fail.com/", - "api_key": test_api_key, - "headers": test_headers, - }, - mode="chat", - ) + with socket.socket() as reserved: + reserved.bind(("127.0.0.1", 0)) + api_base = f"http://127.0.0.1:{reserved.getsockname()[1]}/" + + response = await ahealth_check( + model_params={ + "model": "databricks/dbrx-instruct", + "api_base": api_base, + "api_key": test_api_key, + "headers": test_headers, + }, + mode="chat", + ) - # Should have error and raw_request_typed_dict assert "error" in response assert "raw_request_typed_dict" in response @@ -243,22 +246,15 @@ async def test_ahealth_check_failure_masks_raw_request_headers(): headers = raw_request_dict["raw_request_headers"] assert headers is not None - # Security check: Authorization header should be masked, not show full key - if "Authorization" in headers: - auth_header = headers["Authorization"] - # Should be masked (e.g., "Be****90" or similar) - assert auth_header != f"Bearer {test_api_key}", "Authorization header must be masked" - assert auth_header != test_api_key, "API key must not appear in Authorization header" - # Masked headers typically have asterisks or are truncated - assert "*" in auth_header or len(auth_header) < len(f"Bearer {test_api_key}"), ( - f"Authorization header should be masked but got: {auth_header}" - ) + assert "Authorization" in headers + auth_header = headers["Authorization"] + assert auth_header != f"Bearer {test_api_key}", "Authorization header must be masked" + assert auth_header != test_api_key, "API key must not appear in Authorization header" + assert "*" in auth_header or len(auth_header) < len(f"Bearer {test_api_key}"), ( + f"Authorization header should be masked but got: {auth_header}" + ) - # Content-Type should remain unmasked (not sensitive) - if "Content-Type" in headers: - assert headers["Content-Type"] == "application/json" - - print(f"Masked Authorization header: {headers.get('Authorization', 'NOT FOUND')}") + assert headers["Content-Type"] == "application/json" @pytest.mark.asyncio diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index ae9b30d862b..75d3a23e012 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -9,7 +9,7 @@ import subprocess import sys import threading import time -import traceback +from collections.abc import Mapping from concurrent.futures import Future, wait from pathlib import Path from typing import Final @@ -448,35 +448,46 @@ class NeedsToleranceUpdateError(Exception): # test_tokenizers() -def test_encoding_and_decoding(): - try: - sample_text = "Hellö World, this is my input string!" - # openai encoding + decoding - openai_tokens = encode(model="gpt-3.5-turbo", text=sample_text) - openai_text = decode(model="gpt-3.5-turbo", tokens=openai_tokens) +def test_encoding_and_decoding(tmp_path: Path): + sample_text = "Hellö World, this is my input string!" - assert openai_text == sample_text + # openai encoding + decoding + openai_tokens = encode(model="gpt-3.5-turbo", text=sample_text) + openai_text = decode(model="gpt-3.5-turbo", tokens=openai_tokens) - # claude encoding + decoding - claude_tokens = encode(model="claude-3-5-haiku-20241022", text=sample_text) + assert openai_text == sample_text - claude_text = decode(model="claude-3-5-haiku-20241022", tokens=claude_tokens) + # claude encoding + decoding + claude_tokens = encode(model="claude-3-5-haiku-20241022", text=sample_text) - assert claude_text == sample_text + claude_text = decode(model="claude-3-5-haiku-20241022", tokens=claude_tokens) - # cohere encoding + decoding - cohere_tokens = encode(model="command-nightly", text=sample_text) - cohere_text = decode(model="command-nightly", tokens=cohere_tokens) + assert claude_text == sample_text - assert cohere_text == sample_text + # cohere encoding + decoding + cohere_tokens = encode(model="command-nightly", text=sample_text) + cohere_text = decode(model="command-nightly", tokens=cohere_tokens) - # llama2 encoding + decoding - llama2_tokens = encode(model="meta-llama/Llama-2-7b-chat", text=sample_text) - llama2_text = decode(model="meta-llama/Llama-2-7b-chat", tokens=llama2_tokens) + assert cohere_text == sample_text - assert llama2_text == sample_text - except Exception as e: - pytest.fail(f"An exception occured: {e}\n{traceback.format_exc()}") + # llama2 encoding + decoding + words = sample_text.split() + result = _run_in_memory_hub( + HUB_ROUND_TRIP_SCRIPT, + { + "hf-internal-testing/llama-tokenizer": _word_level_tokenizer_json( + pre_tokenizers.WhitespaceSplit(), + vocab={"[UNK]": 0, **{word: i + 1 for i, word in enumerate(words)}}, + ) + }, + sample_text, + tmp_path, + ) + + assert result["decoded"] == sample_text + assert result["requested"] == ["hf-internal-testing/llama-tokenizer"] + assert len(result["tokens"]) == len(words) + assert len(result["tokens"]) != len(encode(model="gpt-3.5-turbo", text=sample_text)) # test_encoding_and_decoding() @@ -1447,7 +1458,7 @@ def test_high_detail_image_token_upper_bound_is_reached_by_the_largest_high_res_ assert calculate_img_tokens(_png_data_url(1, 1), mode="high") < high_detail_image_token_upper_bound() -HUB_TOKENIZER_SCRIPT: Final = """ +HUB_SETUP_SCRIPT: Final = """ import json import sys sys.path.insert(0, sys.argv[1]) @@ -1466,6 +1477,9 @@ def handle(request): headers = {"content-length": str(len(payload)), "etag": '"fixture"', "x-repo-commit": "a" * 40} return httpx.Response(200, headers=headers, content=payload if request.method == "GET" else b"") huggingface_hub.set_client_factory(lambda: httpx.Client(transport=httpx.MockTransport(handle))) +""" + +HUB_TOKENIZER_SCRIPT: Final = HUB_SETUP_SCRIPT + """ litellm.cohere_models = {"command-r-v1"} litellm.anthropic_models = {"claude-2"} custom = litellm.create_pretrained_tokenizer("Xenova/llama-3-tokenizer") @@ -1479,34 +1493,32 @@ print(json.dumps({ })) """ +HUB_ROUND_TRIP_SCRIPT: Final = HUB_SETUP_SCRIPT + """ +tokens = litellm.encode(model="meta-llama/Llama-2-7b-chat", text=text) +print(json.dumps({"tokens": tokens, "decoded": litellm.decode(model="meta-llama/Llama-2-7b-chat", tokens=tokens), "requested": sorted(set(requested))})) +""" -def _word_level_tokenizer_json(pre_tokenizer: pre_tokenizers.PreTokenizer) -> str: - tokenizer: Final = Tokenizer(models.WordLevel(vocab={"[UNK]": 0}, unk_token="[UNK]")) + +def _word_level_tokenizer_json( + pre_tokenizer: pre_tokenizers.PreTokenizer, vocab: Mapping[str, int] | None = None +) -> str: + tokenizer: Final = Tokenizer( + models.WordLevel(vocab=dict(vocab) if vocab is not None else {"[UNK]": 0}, unk_token="[UNK]") + ) tokenizer.pre_tokenizer = pre_tokenizer return tokenizer.to_str() -def test_token_counter_uses_the_tokenizer_of_each_model_family_and_of_a_custom_tokenizer(tmp_path: Path) -> None: - sample: Final = "Tokenizers disagree: anthropic, tiktoken; llama-2 & llama-3!" - served: Final = { - "hf-internal-testing/llama-tokenizer": _word_level_tokenizer_json(pre_tokenizers.WhitespaceSplit()), - "Xenova/llama-3-tokenizer": _word_level_tokenizer_json(pre_tokenizers.Split(Regex("."), "isolated")), - "Xenova/c4ai-command-r-v01-tokenizer": _word_level_tokenizer_json(pre_tokenizers.Whitespace()), - } - expected: Final = {repo: len(Tokenizer.from_str(payload).encode(sample).ids) for repo, payload in served.items()} - anthropic_count: Final = len(Tokenizer.from_str(claude_json_str).encode(sample).ids) - tiktoken_count: Final = litellm.token_counter(model="gpt-3.5-turbo", text=sample) - assert len({*expected.values(), anthropic_count, tiktoken_count}) == len(expected) + 2 - +def _run_in_memory_hub(script: str, served: dict[str, str], text: str, tmp_path: Path) -> dict: result: Final = subprocess.run( [ sys.executable, "-I", "-c", - HUB_TOKENIZER_SCRIPT, + script, str(Path(litellm.__file__).parent.parent), json.dumps(served), - sample, + text, ], capture_output=True, text=True, @@ -1522,7 +1534,22 @@ def test_token_counter_uses_the_tokenizer_of_each_model_family_and_of_a_custom_t ) assert result.returncode == 0, result.stdout + result.stderr - counts: Final = json.loads(result.stdout.strip().splitlines()[-1]) + return json.loads(result.stdout.strip().splitlines()[-1]) + + +def test_token_counter_uses_the_tokenizer_of_each_model_family_and_of_a_custom_tokenizer(tmp_path: Path) -> None: + sample: Final = "Tokenizers disagree: anthropic, tiktoken; llama-2 & llama-3!" + served: Final = { + "hf-internal-testing/llama-tokenizer": _word_level_tokenizer_json(pre_tokenizers.WhitespaceSplit()), + "Xenova/llama-3-tokenizer": _word_level_tokenizer_json(pre_tokenizers.Split(Regex("."), "isolated")), + "Xenova/c4ai-command-r-v01-tokenizer": _word_level_tokenizer_json(pre_tokenizers.Whitespace()), + } + expected: Final = {repo: len(Tokenizer.from_str(payload).encode(sample).ids) for repo, payload in served.items()} + anthropic_count: Final = len(Tokenizer.from_str(claude_json_str).encode(sample).ids) + tiktoken_count: Final = litellm.token_counter(model="gpt-3.5-turbo", text=sample) + assert len({*expected.values(), anthropic_count, tiktoken_count}) == len(expected) + 2 + + counts: Final = _run_in_memory_hub(HUB_TOKENIZER_SCRIPT, served, sample, tmp_path) assert counts == { "llama2": expected["hf-internal-testing/llama-tokenizer"], "llama3": expected["Xenova/llama-3-tokenizer"], diff --git a/tests/unit/llms/anthropic/batches/test_handler.py b/tests/unit/llms/anthropic/batches/test_handler.py index 6fde6350127..28b84123482 100644 --- a/tests/unit/llms/anthropic/batches/test_handler.py +++ b/tests/unit/llms/anthropic/batches/test_handler.py @@ -10,8 +10,7 @@ env) - and assert exactly which seam fired, with what URL/headers, and that the parsed result is the LiteLLMBatch the transform produced. The sync ``retrieve_batch`` dispatch (``_is_async`` true -> coroutine, false -> -asyncio.run) is exercised directly, mirroring the dispatch-contract discipline in -tests/test_litellm/batches/test_main.py. +asyncio.run) is exercised directly. """ from unittest.mock import AsyncMock, MagicMock, patch diff --git a/tests/unit/llms/anthropic/test_anthropic_output_format_filter.py b/tests/unit/llms/anthropic/test_anthropic_output_format_filter.py index 90cba035760..b2850192f91 100644 --- a/tests/unit/llms/anthropic/test_anthropic_output_format_filter.py +++ b/tests/unit/llms/anthropic/test_anthropic_output_format_filter.py @@ -1,9 +1,7 @@ """ Coverage for filter_anthropic_output_schema's array/object constraint stripping. -Mirrors tests/litellm/llms/anthropic/test_anthropic_schema_filter.py, but lives -under tests/test_litellm/ so the coverage-uploading CI job exercises the stripped -keyword handling (uniqueItems / contains / minProperties / maxProperties plus +Exercises the stripped keyword handling (uniqueItems / contains / minProperties / maxProperties plus multipleOf / patternProperties / propertyNames / dependentRequired / dependentSchemas / unevaluatedProperties / if / then / else / not / prefixItems), the ``uniqueItems: false`` branch, the oneOf to anyOf rewrite, and the