diff --git a/.circleci/scripts/classify_changes.sh b/.circleci/scripts/classify_changes.sh index 01bc8290199..ad265a5e39f 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.circleci/scripts/classify_changes.sh @@ -31,7 +31,7 @@ while IFS= read -r file || [ -n "$file" ]; do case "$file" in model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json) has_cost_map=true ;; - tests/test_litellm/* | tests/proxy_unit_tests/*) : ;; + tests/test_litellm/* | tests/proxy_unit_tests/* | tests/unit/proxy/*) : ;; *) outside_cost_map_set=true ;; esac done diff --git a/.circleci/scripts/unit_selection.sh b/.circleci/scripts/unit_selection.sh new file mode 100755 index 00000000000..bac7fdad842 --- /dev/null +++ b/.circleci/scripts/unit_selection.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +set -euo pipefail + +flag="${1:?usage: unit_selection.sh }" + +legacy_flags=( + caching-local + enterprise-package + enterprise-routing + mcp-integration + proxy-db-auth-checks + proxy-db-budgets + proxy-db-custom-logging + proxy-db-db-and-spend + proxy-db-endpoints-and-responses + proxy-db-guardrails-hooks + proxy-db-jwt-and-keys + proxy-db-key-generation + proxy-db-logging-misc + proxy-db-proxy-runtime + proxy-db-proxy-server-core + proxy-db-proxy-utils + proxy-extras + proxy-infra +) + +legacy_paths() { + case "$1" in + caching-local) echo tests/unit/caching ;; + enterprise-package) + echo tests/unit/enterprise/integrations + echo tests/unit/enterprise/proxy/auth + echo tests/unit/enterprise/proxy/guardrails + echo tests/unit/enterprise/proxy/hooks + echo tests/unit/enterprise/proxy/management_endpoints + echo tests/unit/enterprise/proxy/test_audit_logging_endpoints.py + echo tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py ;; + enterprise-routing) + echo tests/unit/enterprise/enterprise_callbacks/send_emails + echo tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py + echo tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py + echo tests/unit/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py + echo tests/unit/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py + echo tests/unit/enterprise/proxy/test_batch_update_db_managed_output_file_id.py + echo tests/unit/enterprise/proxy/test_deleted_file_returns_403_not_404.py + echo tests/unit/enterprise/proxy/test_enterprise_routes.py + echo tests/unit/enterprise/proxy/test_file_deletion_blocking.py + echo tests/unit/enterprise/proxy/test_managed_files_access_check.py + echo tests/unit/enterprise/proxy/test_managed_files_hook.py ;; + mcp-integration) + echo tests/unit/proxy/_experimental/mcp_server + echo tests/unit/responses/mcp + echo tests/integration/mcp_sdk1 ;; + proxy-db-auth-checks) + echo tests/unit/proxy/auth/test_auth_checks.py + echo tests/unit/proxy/auth/test_user_api_key_auth.py + echo tests/unit/proxy/test_deprecated_key_grace_period.py ;; + proxy-db-budgets) + echo tests/unit/proxy/auth/test_default_end_user_budget_simple.py + echo tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py + echo tests/unit/proxy/test_zero_cost_model_budget_bypass.py ;; + proxy-db-custom-logging) + echo tests/unit/proxy/test_custom_callback_input.py + echo tests/unit/proxy/test_custom_logger_s3_gcs.py ;; + proxy-db-db-and-spend) + echo tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py + echo tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py + echo tests/unit/proxy/db/test_update_daily_tag_spend.py + echo tests/unit/proxy/test_db_schema_changes.py + echo tests/unit/proxy/test_prisma_client_backoff_retry.py + echo tests/unit/proxy/test_update_spend.py + echo tests/unit/skills/test_skills_db.py ;; + proxy-db-endpoints-and-responses) + echo tests/unit/proxy/auth/test_models_fallback_endpoint.py + echo tests/unit/proxy/common_utils/test_check_batch_cost.py + echo tests/unit/proxy/common_utils/test_check_responses_cost.py + echo tests/unit/proxy/common_utils/test_realtime_cache.py + echo tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py + echo tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py + echo tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py + echo tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py + echo tests/unit/proxy/response_polling/test_response_polling_handler.py + echo tests/unit/proxy/test_custom_tokenizer_bug.py + echo tests/unit/proxy/test_get_favicon.py + echo tests/unit/proxy/test_get_image.py + echo tests/unit/proxy/test_prompt_test_endpoint.py + echo tests/unit/proxy/test_reducto_ocr_route.py + echo tests/unit/proxy/test_response_polling_pre_call_checks.py + echo tests/unit/proxy/test_ui_path_detection.py ;; + proxy-db-guardrails-hooks) + echo tests/unit/proxy/hooks/test_banned_keyword_list.py + echo tests/unit/proxy/test_proxy_setting_guardrails.py + echo tests/unit/proxy/test_unit_test_proxy_hooks.py ;; + proxy-db-jwt-and-keys) + echo tests/unit/proxy/auth/test_jwt.py + echo tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py + echo tests/unit/proxy/test_proxy_custom_auth.py ;; + proxy-db-key-generation) echo tests/unit/proxy/management_endpoints/test_key_generate_prisma.py ;; + proxy-db-logging-misc) + echo tests/unit/proxy/management_helpers/test_audit_logs_proxy.py + echo tests/unit/proxy/spend_tracking/test_search_api_logging.py + echo tests/unit/proxy/test_proxy_reject_logging.py ;; + proxy-db-proxy-runtime) + echo tests/unit/proxy/auth/test_multipart_bypass_repro.py + echo tests/unit/proxy/auth/test_proxy_routes.py + echo tests/unit/proxy/middleware/test_request_size_limit_middleware.py + echo tests/unit/proxy/test_proxy_config_unit_test.py + echo tests/unit/proxy/test_proxy_pass_user_config.py + echo tests/unit/proxy/test_proxy_token_counter.py + echo tests/unit/proxy/test_server_root_path.py ;; + proxy-db-proxy-server-core) + echo tests/unit/proxy/test_aproxy_startup.py + echo tests/unit/proxy/test_proxy_server.py ;; + proxy-db-proxy-utils) echo tests/unit/proxy/test_proxy_utils.py ;; + proxy-extras) echo tests/unit/litellm_proxy_extras ;; + proxy-infra) echo tests/unit/gateway ;; + *) echo "unit_selection.sh: unknown flag $1" >&2; exit 1 ;; + esac +} + +expand() { + while read -r path; do + if [ -d "$path" ]; then + find "$path" -name 'test_*.py' + elif [ -f "$path" ]; then + echo "$path" + else + echo "unit_selection.sh: $path does not exist" >&2 + exit 1 + fi + done +} + +if [ "$flag" = unit ]; then + comm -23 \ + <(find tests/unit -name 'test_*.py' | sort) \ + <(for legacy in "${legacy_flags[@]}"; do legacy_paths "$legacy"; done | expand | sort) + exit 0 +fi + +legacy_paths "$flag" | expand | sort diff --git a/.circleci/tests.yml b/.circleci/tests.yml index dfdecfdd347..05cb7677960 100644 --- a/.circleci/tests.yml +++ b/.circleci/tests.yml @@ -165,21 +165,27 @@ commands: jobs: unit: parameters: - tests_path: - type: string - default: tests/unit flag: type: string default: unit shards: type: integer default: 6 + workers: + type: integer + default: 4 + dist: + type: string + default: loadscope base_ref: type: string default: "" pull_request_url: type: string default: "" + legacy_mcp_peer: + type: boolean + default: false machine: image: ubuntu-2204:2024.04.1 resource_class: large @@ -194,15 +200,26 @@ jobs: base_ref: << parameters.base_ref >> pull_request_url: << parameters.pull_request_url >> - setup_test_deps + - when: + condition: << parameters.legacy_mcp_peer >> + steps: + - run: + name: Install MCP SDK1 peer + command: | + uv venv --python 3.12 .venv-mcp-peer + uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1' + echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV" - run: - name: "Run << parameters.tests_path >> shard" + name: "Run << parameters.flag >> shard" no_output_timeout: 20m command: | mkdir -p test-results/<< parameters.flag >> - mapfile -t files < <(find << parameters.tests_path >> -name 'test_*.py' | sort | circleci tests split --split-by=timings --timings-type=filename) - if [ "${#files[@]}" -eq 0 ]; then echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.tests_path >> files; nothing to run"; exit 0; fi + mapfile -t files < <(bash .circleci/scripts/unit_selection.sh << parameters.flag >> | circleci tests split --split-by=timings --timings-type=filename) + if [ "${#files[@]}" -eq 0 ]; then echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.flag >> files; nothing to run"; exit 0; fi + xdist_args=() + if [ "<< parameters.workers >>" -gt 0 ]; then xdist_args=(-n << parameters.workers >> --dist=<< parameters.dist >>); fi set +e - uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml + uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 "${xdist_args[@]}" --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml status=$? set -e if [ "$status" -eq 5 ]; then echo "pytest collected no tests from the shard; passing"; exit 0; fi @@ -286,6 +303,57 @@ workflows: - unit: base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> + - unit: + name: unit-<< matrix.flag >> + shards: 1 + workers: 2 + matrix: + parameters: + flag: [caching-local, proxy-extras, enterprise-routing] + base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> + pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> + - unit: + name: unit-mcp-integration + flag: mcp-integration + shards: 1 + workers: 2 + legacy_mcp_peer: true + base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> + pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> + - unit: + name: unit-<< matrix.flag >> + shards: 1 + matrix: + parameters: + flag: + - enterprise-package + - proxy-infra + - proxy-db-auth-checks + - proxy-db-jwt-and-keys + - proxy-db-proxy-server-core + - proxy-db-proxy-runtime + - proxy-db-custom-logging + - proxy-db-logging-misc + - proxy-db-db-and-spend + - proxy-db-guardrails-hooks + - proxy-db-budgets + - proxy-db-endpoints-and-responses + base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> + pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> + - unit: + name: unit-proxy-db-proxy-utils + flag: proxy-db-proxy-utils + shards: 1 + dist: worksteal + base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> + pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> + - unit: + name: unit-proxy-db-key-generation + flag: proxy-db-key-generation + shards: 1 + workers: 0 + base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> + pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> - documentation - integration: name: integration-<< matrix.suite >> diff --git a/.github/scripts/assert_ci_coverage.py b/.github/scripts/assert_ci_coverage.py index 2e008fe7ade..01a01b1034b 100644 --- a/.github/scripts/assert_ci_coverage.py +++ b/.github/scripts/assert_ci_coverage.py @@ -34,7 +34,6 @@ GLOB_CHARS = frozenset("*?") # tests has to be named by some shard or it runs nowhere. A child listed here is # itself decomposed one level deeper and is checked through its own entry. SHARDED_ROOTS: tuple[str, ...] = ( - "tests/proxy_unit_tests", "tests/test_litellm", "tests/test_litellm/proxy", ) @@ -120,6 +119,13 @@ def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]: ) +def _unit_selection_tokens(repo_root: pathlib.Path = REPO_ROOT) -> frozenset[str]: + script: Final = repo_root / ".circleci/scripts/unit_selection.sh" + if not script.is_file(): + return frozenset() + return frozenset(match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(_uncommented(script.read_text()))) + + def _built_dockerfile_tokens(scalars: Iterable[Scalar]) -> frozenset[str]: return frozenset( match.group(0) @@ -611,7 +617,10 @@ def main() -> int: scalars = _all_scalars() integration_paths, ownership_findings = _integration_ownership() - test_findings = _uncovered_tests(allowlist, _invoked_test_tokens(scalars) | integration_paths) + ownership_findings + test_findings = ( + _uncovered_tests(allowlist, _invoked_test_tokens(scalars) | _unit_selection_tokens() | integration_paths) + + ownership_findings + ) dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars)) stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles()) diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index 8faddd11df8..ef1dc53b4a6 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -13,6 +13,15 @@ on: have its path existence-checked like any other token. required: true type: string + fork-flag: + description: >- + Codecov flag of the `.circleci/tests.yml` job that now owns part of + this shard. CircleCI does not run on pull requests from forks, so on + those events this shard also runs the files + `.circleci/scripts/unit_selection.sh` lists for the flag. + required: false + type: string + default: "" workers: description: "Number of pytest-xdist workers" required: false @@ -92,6 +101,7 @@ jobs: pull-requests: read outputs: decision: ${{ steps.changes.outputs.decision }} + has-coverage: ${{ steps.tests.outputs.has-coverage }} steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 @@ -160,10 +170,13 @@ jobs: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - name: Run tests + id: tests if: steps.changes.outputs.decision != 'skip' timeout-minutes: ${{ inputs.timeout-minutes }} env: TEST_PATH: ${{ inputs.test-path }} + FORK_FLAG: ${{ inputs.fork-flag }} + IS_FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} MAX_FAILURES: ${{ inputs.max-failures }} WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} @@ -171,9 +184,18 @@ jobs: DIST: ${{ inputs.dist }} COVERAGE_CORE: sysmon run: | + echo "has-coverage=false" >> "$GITHUB_OUTPUT" + selection="${TEST_PATH}" + if [ "${IS_FORK}" = "true" ] && [ -n "${FORK_FLAG}" ]; then + selection="${TEST_PATH} $(bash .circleci/scripts/unit_selection.sh "${FORK_FLAG}" | tr '\n' ' ')" + fi + if [ -z "${selection// /}" ]; then + echo "shard selection is empty on this event (CircleCI flag ${FORK_FLAG:-none} owns it); nothing to run" + exit 0 + fi pytest_args=() existing_paths=0 - for token in ${TEST_PATH:?}; do + for token in ${selection}; do case "${token}" in -*) pytest_args+=("${token}") ;; *) @@ -187,7 +209,7 @@ jobs: esac done if [ "${existing_paths}" -eq 0 ]; then - echo "No path in TEST_PATH exists (${TEST_PATH}); nothing to run" + echo "No path in the selection exists (${selection}); nothing to run" exit 0 fi xdist_args=() @@ -209,8 +231,11 @@ jobs: --cov-config=pyproject.toml status=$? set -e + if [ -f coverage.xml ]; then + echo "has-coverage=true" >> "$GITHUB_OUTPUT" + fi if [ "$status" -eq 5 ]; then - echo "pytest collected no tests from ${TEST_PATH}; passing" + echo "pytest collected no tests from ${selection}; passing" exit 0 fi exit "$status" @@ -226,7 +251,7 @@ jobs: upload-coverage: name: Upload coverage to Codecov needs: run - if: always() && needs.run.outputs.decision != 'skip' + if: always() && needs.run.outputs.decision != 'skip' && needs.run.outputs.has-coverage == 'true' runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index 4af7a161984..86b385d91a7 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -20,6 +20,12 @@ concurrency: # rather than alphabetical letter ranges. Adding a new test file means adding it # to whichever group it belongs to, not reshuffling slices. # +# `.circleci/tests.yml` runs each group's files on same-repo events under the +# `proxy-db-` Codecov flag; `.circleci/scripts/unit_selection.sh` holds +# the file lists. CircleCI does not build pull requests from forks, so `fork-flag` +# makes the shard run that list there. `test-path` keeps the files that still +# reach real providers and never left tests/proxy_unit_tests. +# # Design targets: # * Every shard runs in <= 7 minutes of wall-clock on the default runner. # Most of a shard's time is pytest plugin load + xdist worker imports + @@ -58,7 +64,7 @@ jobs: proxy-db: needs: assert-shard-coverage # Display only the semantic shard name in the checks UI instead of GHA's - # default "proxy-db (key-generation, tests/proxy_unit_tests/โ€ฆ, 0, loadscope, 20)" + # default "proxy-db (key-generation, tests/unit/proxy/โ€ฆ, 0, loadscope, 20)" # which includes every matrix field and gets truncated past the test-path. name: ${{ matrix.test-group }} permissions: @@ -71,132 +77,93 @@ jobs: include: # Must run serially โ€” event-loop conflict with the logging worker. - test-group: key-generation - test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py" + test-path: "" + fork-flag: proxy-db-key-generation workers: 0 dist: loadscope timeout: 20 # ---- auth: split into 2 shards ---- - test-group: auth-checks - test-path: >- - tests/proxy_unit_tests/test_auth_checks.py - tests/proxy_unit_tests/test_user_api_key_auth.py - tests/proxy_unit_tests/test_deprecated_key_grace_period.py + test-path: "" + fork-flag: proxy-db-auth-checks workers: 4 dist: loadscope timeout: 15 - test-group: jwt-and-keys - test-path: >- - tests/proxy_unit_tests/test_jwt.py - tests/proxy_unit_tests/test_jwt_key_mapping.py - tests/proxy_unit_tests/test_proxy_custom_auth.py - tests/proxy_unit_tests/test_key_generate_dynamodb.py + test-path: "" + fork-flag: proxy-db-jwt-and-keys workers: 4 dist: loadscope timeout: 15 # ---- test_proxy_utils.py, single shard, worksteal distribution ---- - test-group: proxy-utils - test-path: "tests/proxy_unit_tests/test_proxy_utils.py" + test-path: "" + fork-flag: proxy-db-proxy-utils workers: 4 dist: worksteal timeout: 15 # ---- proxy server: split into 2 shards ---- - test-group: proxy-server-core - test-path: >- - tests/proxy_unit_tests/test_proxy_server.py - tests/proxy_unit_tests/test_aproxy_startup.py + test-path: "tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py" + fork-flag: proxy-db-proxy-server-core workers: 4 dist: loadscope timeout: 15 - test-group: proxy-runtime - test-path: >- - tests/proxy_unit_tests/test_proxy_config_unit_test.py - tests/proxy_unit_tests/test_proxy_routes.py - tests/proxy_unit_tests/test_server_root_path.py - tests/proxy_unit_tests/test_proxy_pass_user_config.py - tests/proxy_unit_tests/test_proxy_token_counter.py - tests/proxy_unit_tests/test_request_size_limit_middleware.py - tests/proxy_unit_tests/test_multipart_bypass_repro.py + test-path: "" + fork-flag: proxy-db-proxy-runtime workers: 4 dist: loadscope timeout: 15 # ---- logging: split into 2 shards ---- - test-group: custom-logging - test-path: >- - tests/proxy_unit_tests/test_custom_callback_input.py - tests/proxy_unit_tests/test_custom_logger_s3_gcs.py - tests/proxy_unit_tests/test_proxy_custom_logger.py + test-path: "tests/proxy_unit_tests/test_proxy_custom_logger.py" + fork-flag: proxy-db-custom-logging workers: 4 dist: loadscope timeout: 15 - test-group: logging-misc - test-path: >- - tests/proxy_unit_tests/test_proxy_reject_logging.py - tests/proxy_unit_tests/test_audit_logs_proxy.py - tests/proxy_unit_tests/test_search_api_logging.py + test-path: "" + fork-flag: proxy-db-logging-misc workers: 4 dist: loadscope timeout: 15 - test-group: db-and-spend - test-path: >- - tests/proxy_unit_tests/test_prisma_client_backoff_retry.py - tests/proxy_unit_tests/test_db_schema_changes.py - tests/proxy_unit_tests/test_e2e_pod_lock_manager.py - tests/proxy_unit_tests/test_skills_db.py - tests/proxy_unit_tests/test_update_daily_tag_spend.py - tests/proxy_unit_tests/test_update_spend.py - tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py + test-path: "" + fork-flag: proxy-db-db-and-spend workers: 4 dist: loadscope timeout: 15 # ---- guardrails + budget + hooks: split into 2 ---- - test-group: guardrails-hooks - test-path: >- - tests/proxy_unit_tests/test_proxy_setting_guardrails.py - tests/proxy_unit_tests/test_banned_keyword_list.py - tests/proxy_unit_tests/test_unit_test_proxy_hooks.py + test-path: "" + fork-flag: proxy-db-guardrails-hooks workers: 4 dist: loadscope timeout: 15 - test-group: budgets - test-path: >- - tests/proxy_unit_tests/test_default_end_user_budget_simple.py - tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py - tests/proxy_unit_tests/test_zero_cost_model_budget_bypass.py + test-path: "" + fork-flag: proxy-db-budgets workers: 4 dist: loadscope timeout: 15 - test-group: endpoints-and-responses - test-path: >- - tests/proxy_unit_tests/test_blog_posts_endpoint.py - tests/proxy_unit_tests/test_models_fallback_endpoint.py - tests/proxy_unit_tests/test_google_endpoint_routing.py - tests/proxy_unit_tests/test_google_gemini_proxy_request.py - tests/proxy_unit_tests/test_gemini_agents_endpoints.py - tests/proxy_unit_tests/test_get_favicon.py - tests/proxy_unit_tests/test_get_image.py - tests/proxy_unit_tests/test_reducto_ocr_route.py - tests/proxy_unit_tests/test_ui_path_detection.py - tests/proxy_unit_tests/test_prompt_test_endpoint.py - tests/proxy_unit_tests/test_check_batch_cost.py - tests/proxy_unit_tests/test_check_responses_cost.py - tests/proxy_unit_tests/test_response_polling_handler.py - tests/proxy_unit_tests/test_response_polling_pre_call_checks.py - tests/proxy_unit_tests/test_realtime_cache.py - tests/proxy_unit_tests/test_proxy_exception_mapping.py - tests/proxy_unit_tests/test_custom_tokenizer_bug.py + test-path: "tests/proxy_unit_tests/test_proxy_exception_mapping.py" + fork-flag: proxy-db-endpoints-and-responses workers: 4 dist: loadscope timeout: 15 uses: ./.github/workflows/_test-unit-base.yml with: test-path: ${{ matrix.test-path }} + fork-flag: ${{ matrix.fork-flag }} workers: ${{ matrix.workers }} reruns: 2 timeout-minutes: ${{ matrix.timeout }} diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 4580ad17a19..082178caa5c 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -31,10 +31,14 @@ concurrency: # number, so a partially-specified entry would fail the call rather than fall # back to the default. # -# tests/proxy_unit_tests keeps its own caller (test-unit-proxy-db.yml): it is -# already a matrix and carries a shard-coverage guard that reads that file by -# name. Folding it in here is a follow-up, together with generalising that guard -# into assert_ci_coverage.py. +# tests/unit/proxy keeps its own caller (test-unit-proxy-db.yml): it is already +# a matrix and carries a shard-coverage guard that reads that file by name. +# Folding it in here is a follow-up, together with generalising that guard into +# assert_ci_coverage.py. +# +# `fork-flag` names the `.circleci/tests.yml` job that now runs part of the +# shard under the same Codecov flag. CircleCI does not build pull requests from +# forks, so the shard still runs those files there and skips them elsewhere. jobs: unit: name: ${{ matrix.shard }} @@ -49,6 +53,7 @@ jobs: - shard: mcp-integration artifact-name: mcp-integration test-path: "tests/mcp_tests tests/test_litellm/experimental_mcp_client" + fork-flag: mcp-integration workers: 2 reruns: 0 timeout-minutes: 20 @@ -65,10 +70,10 @@ jobs: - shard: enterprise-routing artifact-name: enterprise-routing test-path: >- - tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils tests/test_litellm/router_strategy + fork-flag: enterprise-routing workers: 2 reruns: 2 timeout-minutes: 20 @@ -199,7 +204,7 @@ jobs: tests/test_litellm/proxy/types_utils tests/test_litellm/proxy/logging_endpoints tests/test_litellm/proxy/test_*.py - tests/test_gateway + fork-flag: proxy-infra workers: 4 reruns: 2 timeout-minutes: 20 @@ -207,11 +212,8 @@ jobs: - shard: caching-local artifact-name: caching-local - test-path: >- - tests/local_testing/test_cache_preset_key.py - tests/local_testing/test_caching_handler.py - tests/local_testing/test_responses_stream_cache_keys.py - tests/local_testing/test_unit_test_caching.py + test-path: "" + fork-flag: caching-local workers: 2 reruns: 2 timeout-minutes: 20 @@ -219,7 +221,8 @@ jobs: - shard: proxy-extras artifact-name: proxy-extras - test-path: "tests/litellm-proxy-extras" + test-path: "" + fork-flag: proxy-extras workers: 2 reruns: 2 timeout-minutes: 20 @@ -227,7 +230,8 @@ jobs: - shard: enterprise-package artifact-name: enterprise-package - test-path: "tests/enterprise" + test-path: "" + fork-flag: enterprise-package workers: 4 reruns: 2 timeout-minutes: 20 @@ -246,6 +250,7 @@ jobs: uses: ./.github/workflows/_test-unit-base.yml with: test-path: ${{ matrix.test-path }} + fork-flag: ${{ matrix.fork-flag || '' }} workers: ${{ matrix.workers }} reruns: ${{ matrix.reruns }} timeout-minutes: ${{ matrix.timeout-minutes }} diff --git a/Makefile b/Makefile index ab7fab6aa99..eba332de781 100644 --- a/Makefile +++ b/Makefile @@ -51,8 +51,8 @@ help: @echo " make test-unit-core-utils - Run core utils tests (~32 files)" @echo " make test-unit-other - Run other tests (caching, responses, etc., ~69 files)" @echo " make test-unit-root - Run root-level tests (~34 files)" - @echo " make test-proxy-unit-a - Run proxy_unit_tests (a-o, ~20 files)" - @echo " make test-proxy-unit-b - Run proxy_unit_tests (p-z, ~28 files)" + @echo " make test-proxy-unit-a - Run tests/unit/proxy (a-o)" + @echo " make test-proxy-unit-b - Run tests/unit/proxy (p-z)" @echo " make test-integration - Run integration tests" @echo " make test-unit-helm - Run helm unit tests" @echo " make test-rust-extension - Build the Rust extension and run its public Python tests" @@ -332,17 +332,17 @@ test-unit-core-utils: install-test-deps $(UV_RUN) pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20 test-unit-other: install-test-deps - $(UV_RUN) pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/unit/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20 test-unit-root: install-test-deps $(UV_RUN) pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20 -# Proxy unit tests (tests/proxy_unit_tests split alphabetically) +# Proxy unit tests (tests/unit/proxy split alphabetically) test-proxy-unit-a: install-test-deps - $(UV_RUN) pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20 + $(UV_RUN) pytest tests/unit/proxy/test_[a-o]*.py --tb=short -vv -n 2 --durations=20 test-proxy-unit-b: install-test-deps - $(UV_RUN) pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20 + $(UV_RUN) pytest tests/unit/proxy/test_[p-z]*.py --tb=short -vv -n 2 --durations=20 test-integration: install-test-deps $(UV_RUN) pytest tests/ -k "not test_litellm" diff --git a/litellm/llms/litellm_proxy/skills/README.md b/litellm/llms/litellm_proxy/skills/README.md index a896aa1166e..ccbd394cddc 100644 --- a/litellm/llms/litellm_proxy/skills/README.md +++ b/litellm/llms/litellm_proxy/skills/README.md @@ -369,7 +369,7 @@ model LiteLLM_SkillsTable { Run the tests: ```bash -pytest tests/proxy_unit_tests/test_skills_db.py -v +pytest tests/unit/skills/test_skills_db.py -v ``` Tests cover: diff --git a/tests/code_coverage_tests/user_api_key_auth_code_coverage.py b/tests/code_coverage_tests/user_api_key_auth_code_coverage.py index a9c2f8ef15f..2f221a7ebe7 100644 --- a/tests/code_coverage_tests/user_api_key_auth_code_coverage.py +++ b/tests/code_coverage_tests/user_api_key_auth_code_coverage.py @@ -31,11 +31,11 @@ def get_function_names_from_file(file_path): def get_all_functions_called_in_tests(base_dir): """ Returns a set of function names that are called in test functions - inside 'local_testing' and 'proxy_unit_tests' directories, + inside 'local_testing' and 'unit/proxy' directories, specifically in files containing the word 'router'. """ called_functions = set() - test_dirs = ["local_testing", "proxy_unit_tests"] + test_dirs = ["local_testing", "unit/proxy"] for test_dir in test_dirs: dir_path = os.path.join(base_dir, test_dir) diff --git a/tests/test_litellm/enterprise/proxy/__init__.py b/tests/integration/mcp_sdk1/__init__.py similarity index 100% rename from tests/test_litellm/enterprise/proxy/__init__.py rename to tests/integration/mcp_sdk1/__init__.py diff --git a/tests/integration/mcp_sdk1/conftest.py b/tests/integration/mcp_sdk1/conftest.py new file mode 100644 index 00000000000..3b265d846ab --- /dev/null +++ b/tests/integration/mcp_sdk1/conftest.py @@ -0,0 +1,53 @@ +import asyncio +import importlib + +import pytest + +import litellm +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + + +@pytest.fixture(scope="session") +def event_loop(): + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + yield loop + loop.close() + + +@pytest.fixture(scope="function", autouse=True) +def setup_and_teardown(): + """ + This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. + """ + importlib.reload(litellm) + import asyncio + + loop = asyncio.get_event_loop_policy().new_event_loop() + asyncio.set_event_loop(loop) + yield + + # Teardown code (executes after the yield point) + # LoggingWorker carries still-queued coroutines onto the next test's loop, where they'd log into that test's callbacks + asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue()) + loop.close() # Close the loop created earlier + asyncio.set_event_loop(None) # Remove the reference to the loop + + +@pytest.fixture(scope="function", autouse=True) +async def drain_logging_worker(): + """ + The logging queue is bound to the running loop, so anything left queued when a test's loop + goes away is carried onto the next test's loop and fires against its callbacks. + """ + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + + yield + + try: + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.clear_queue(), timeout=10) + except asyncio.TimeoutError: + pass + diff --git a/tests/mcp_tests/mcp_server.py b/tests/integration/mcp_sdk1/mcp_math_server.py similarity index 100% rename from tests/mcp_tests/mcp_server.py rename to tests/integration/mcp_sdk1/mcp_math_server.py diff --git a/tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml b/tests/integration/mcp_sdk1/test_config_mcp_e2e.yaml similarity index 91% rename from tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml rename to tests/integration/mcp_sdk1/test_config_mcp_e2e.yaml index 19fad3d1393..29cb8d2726d 100644 --- a/tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml +++ b/tests/integration/mcp_sdk1/test_config_mcp_e2e.yaml @@ -17,7 +17,7 @@ mcp_servers: transport: stdio command: python3 args: - - tests/mcp_tests/mcp_server.py + - tests/integration/mcp_sdk1/mcp_math_server.py allow_all_keys: true # Needed for master_key auth when DB/object_permission not used math_streamable_http: transport: http diff --git a/tests/mcp_tests/test_proxy_mcp_e2e.py b/tests/integration/mcp_sdk1/test_proxy_mcp_e2e.py similarity index 99% rename from tests/mcp_tests/test_proxy_mcp_e2e.py rename to tests/integration/mcp_sdk1/test_proxy_mcp_e2e.py index a730f6c10ee..8fb302855f5 100644 --- a/tests/mcp_tests/test_proxy_mcp_e2e.py +++ b/tests/integration/mcp_sdk1/test_proxy_mcp_e2e.py @@ -35,10 +35,10 @@ from litellm.proxy.proxy_server import ( initialize, ) -CONFIG_TEMPLATE_PATH = Path("tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml") -MCP_SERVER_SCRIPT = Path("tests/mcp_tests/mcp_server.py") +PROJECT_ROOT = Path(__file__).resolve().parents[3] +CONFIG_TEMPLATE_PATH = PROJECT_ROOT / "tests/integration/mcp_sdk1/test_config_mcp_e2e.yaml" +MCP_SERVER_SCRIPT = PROJECT_ROOT / "tests/integration/mcp_sdk1/mcp_math_server.py" MCP_PEER_PYTHON = os.environ.get("MCP_TEST_PEER_PYTHON", sys.executable) -PROJECT_ROOT = Path(__file__).resolve().parents[2] PROXY_START_TIMEOUT = 30 diff --git a/tests/mcp_tests/test_aresponses_api_with_mcp_providers.py b/tests/mcp_tests/test_aresponses_api_with_mcp_providers.py new file mode 100644 index 00000000000..4c9687ccf1a --- /dev/null +++ b/tests/mcp_tests/test_aresponses_api_with_mcp_providers.py @@ -0,0 +1,401 @@ +import logging +import os +import pytest +from mcp.types import Tool as MCPTool +from typing import Any, cast + + +# Import required modules +import litellm +from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler + + +class MockUserAPIKeyAuth: + """Mock UserAPIKeyAuth for testing""" + + def __init__(self): + self.api_key = "test_key" + self.user_id = "test_user" + self.team_id = "test_team" + self.user_email = "test@example.com" + self.max_budget = 100.0 + self.spend = 0.0 + self.models = [] + self.aliases = {} + self.config = {} + self.permissions = {} + self.metadata = {} + self.object_permission_id = "test_permission_id" + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model", + [ + pytest.param("gpt-4o-mini", id="openai"), + pytest.param("claude-haiku-4-5", id="anthropic"), + ], +) +async def test_streaming_responses_api_with_mcp_tools( + model: str, caplog: pytest.LogCaptureFixture +): + """ + Test the streaming responses API with MCP tools when using server_url="litellm_proxy" + + Under the hood the follow occurs + + - MCP: responses called litellm MCP manager.list_tools (MOCKED) + - Request 1: Made to model under test with fetched tools (REAL LLM CALL) + - MCP: Execute tool call from request 1 and returns result (MOCKED) + - Request 2: Made to model under test with fetched tools and tool results (REAL LLM CALL) + + Return the user the result of request 2 + """ + # Skip test if API keys are not set for the respective models + if ("claude" in model.lower() or "anthropic" in model.lower()) and not os.getenv( + "ANTHROPIC_API_KEY" + ): + pytest.skip("ANTHROPIC_API_KEY not set, skipping anthropic model test") + if ("gpt" in model.lower() or "openai" in model.lower()) and not os.getenv( + "OPENAI_API_KEY" + ): + pytest.skip("OPENAI_API_KEY not set, skipping openai model test") + + from unittest.mock import AsyncMock, patch + + print("๐Ÿงช Testing basic streaming with MCP tools...") + + # Mock MCP tools that would be returned from the manager + mock_mcp_tools = [ + MCPTool.model_validate({ + "name": "search_repo", + "description": "Search BerriAI/litellm repository for information", + "inputSchema": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query"} + }, + "required": ["query"], + }, + }, by_name=False) + ] + + # Only mock the MCP-specific operations, let LLM responses be real + with caplog.at_level(logging.ERROR): + with ( + patch.object( + LiteLLM_Proxy_MCP_Handler, + "_get_mcp_tools_from_manager", + new_callable=AsyncMock, + ) as mock_get_tools, + patch.object( + LiteLLM_Proxy_MCP_Handler, + "_execute_tool_calls", + new_callable=AsyncMock, + ) as mock_execute_tools, + ): + # Setup MCP mocks only + mock_get_tools.return_value = (mock_mcp_tools, ["litellm_proxy"]) + + # Create a dynamic mock that will match the actual tool call ID from the LLM response + def mock_execute_tool_calls_side_effect( + tool_calls, user_api_key_auth, **kwargs + ): + """Mock function that returns results matching the actual tool call IDs from the LLM""" + results = [] + for tool_call in tool_calls: + # Extract call_id from the tool call + call_id = None + if isinstance(tool_call, dict): + call_id = tool_call.get("call_id") or tool_call.get("id") + elif hasattr(tool_call, "call_id"): + call_id = tool_call.call_id + elif hasattr(tool_call, "id"): + call_id = tool_call.id + + if call_id: + results.append( + { + "tool_call_id": call_id, + "result": "LiteLLM is a unified interface for 100+ LLMs that translates inputs to provider-specific completion endpoints and provides consistent OpenAI-format output.", + } + ) + return results + + mock_execute_tools.side_effect = mock_execute_tool_calls_side_effect + + # Make the actual call - LLM responses will be real + mcp_tool_config = cast( + Any, + { + "type": "mcp", + "server_url": "litellm_proxy", + "require_approval": "never", + }, + ) + response = await litellm.aresponses( + model=model, + tools=[mcp_tool_config], + tool_choice="required", + input=[ + { + "role": "user", + "type": "message", + "content": "give me a TLDR of what BerriAI/litellm is about", + } + ], + stream=True, + ) + + print(f"๐Ÿ“‹ Response type: {type(response)}") + assert hasattr( + response, "__aiter__" + ), "Response should be an async streaming response" + + # Collect streaming chunks + chunks = [] + async for chunk in response: + chunks.append(chunk) + print(f"๐Ÿ“ฆ Chunk type: {getattr(chunk, 'type', 'unknown')}") + + print(f"๐Ÿ“Š Total chunks received: {len(chunks)}") + + # Verify MCP mocks were called (may be called multiple times in streaming) + assert ( + mock_get_tools.call_count >= 1 + ), f"Expected MCP tools to be fetched at least once, got {mock_get_tools.call_count}" + print(f"MCP tools fetched: {len(mock_mcp_tools)}") + + # Verify we got a response + assert response is not None + assert len(chunks) > 0, "Should have received streaming chunks" + + print("Basic streaming responses API with MCP tools test passed!") + + lite_errors = [ + record + for record in caplog.records + if record.levelno >= logging.ERROR + and ("LiteLLM" in record.name or "LiteLLM" in record.getMessage()) + ] + assert not lite_errors, "Unexpected LiteLLM errors: " + ", ".join( + record.getMessage() for record in lite_errors + ) + + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["gpt-4o-mini"]) +async def test_streaming_mcp_event_order_and_response_id_consistency( + model: str, caplog: pytest.LogCaptureFixture +): + """ + Test that: + 1. Streaming events are emitted in correct order (response.created, response.in_progress, response.output_item.added before MCP events) + 2. All response lifecycle events share the same response ID within a cycle + """ + if ("gpt" in model.lower() or "openai" in model.lower()) and not os.getenv( + "OPENAI_API_KEY" + ): + pytest.skip("OPENAI_API_KEY not set, skipping openai model test") + + from unittest.mock import AsyncMock, patch + + mock_mcp_tools = [ + MCPTool.model_validate({ + "name": "get_weather", + "description": "Get weather for a city", + "inputSchema": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"} + }, + "required": ["city"], + }, + }, by_name=False) + ] + + with caplog.at_level(logging.ERROR): + with ( + patch.object( + LiteLLM_Proxy_MCP_Handler, + "_get_mcp_tools_from_manager", + new_callable=AsyncMock, + ) as mock_get_tools, + patch.object( + LiteLLM_Proxy_MCP_Handler, + "_execute_tool_calls", + new_callable=AsyncMock, + ) as mock_execute_tools, + ): + mock_get_tools.return_value = (mock_mcp_tools, ["litellm_proxy"]) + + def mock_execute_side_effect(tool_calls, user_api_key_auth, **kwargs): + results = [] + for tool_call in tool_calls: + call_id = None + if isinstance(tool_call, dict): + call_id = tool_call.get("call_id") or tool_call.get("id") + elif hasattr(tool_call, "call_id"): + call_id = tool_call.call_id + elif hasattr(tool_call, "id"): + call_id = tool_call.id + if call_id: + results.append( + { + "tool_call_id": call_id, + "result": "Sunny, 72ยฐF", + } + ) + return results + + mock_execute_tools.side_effect = mock_execute_side_effect + + mcp_tool_config = cast( + Any, + { + "type": "mcp", + "server_url": "litellm_proxy", + "require_approval": "never", + }, + ) + + response = await litellm.aresponses( + model=model, + tools=[mcp_tool_config], + input=[ + { + "role": "user", + "type": "message", + "content": "What's the weather in San Francisco?", + } + ], + stream=True, + ) + + events = [] + async for chunk in response: + events.append(chunk) + + assert len(events) > 0, "Should receive streaming events" + + created_idx = next( + ( + i + for i, e in enumerate(events) + if getattr(e, "type", None) == "response.created" + ), + None, + ) + in_progress_idx = next( + ( + i + for i, e in enumerate(events) + if getattr(e, "type", None) == "response.in_progress" + ), + None, + ) + output_item_added_idx = next( + ( + i + for i, e in enumerate(events) + if getattr(e, "type", None) == "response.output_item.added" + ), + None, + ) + mcp_in_progress_idx = next( + ( + i + for i, e in enumerate(events) + if "mcp_list_tools.in_progress" in str(getattr(e, "type", "")) + ), + None, + ) + completed_idx = next( + ( + i + for i, e in enumerate(events) + if getattr(e, "type", None) == "response.completed" + ), + None, + ) + + assert created_idx is not None, "response.created event should be present" + assert ( + in_progress_idx is not None + ), "response.in_progress event should be present" + assert ( + output_item_added_idx is not None + ), "response.output_item.added event should be present" + + assert ( + created_idx < in_progress_idx + ), "response.created should come before response.in_progress" + assert ( + in_progress_idx < output_item_added_idx + ), "response.in_progress should come before response.output_item.added" + + if mcp_in_progress_idx is not None: + assert ( + output_item_added_idx < mcp_in_progress_idx + ), "response.output_item.added should come before response.mcp_list_tools.in_progress" + + response_ids = [] + for i, event in enumerate(events): + event_type = getattr(event, "type", None) + if hasattr(event, "response"): + response_obj = getattr(event, "response", None) + if response_obj and hasattr(response_obj, "id"): + event_type_value = ( + event_type.value + if hasattr(event_type, "value") + else str(event_type) + ) + if any( + x in event_type_value + for x in [ + "response.created", + "response.in_progress", + "response.completed", + ] + ): + response_ids.append((i, event_type_value, response_obj.id)) + + assert ( + len(response_ids) >= 2 + ), f"Should have at least 2 response lifecycle events. Found {len(response_ids)}" + + cycles = [] + current_cycle = [] + current_id = None + + for idx, event_type, resp_id in response_ids: + if current_id is None or resp_id == current_id: + current_cycle.append((idx, event_type, resp_id)) + current_id = resp_id + else: + if current_cycle: + cycles.append(current_cycle) + current_cycle = [(idx, event_type, resp_id)] + current_id = resp_id + if current_cycle: + cycles.append(current_cycle) + + for cycle_num, cycle in enumerate(cycles): + cycle_ids = set(resp_id for _, _, resp_id in cycle) + assert ( + len(cycle_ids) == 1 + ), f"Cycle {cycle_num + 1} should have consistent response ID. Found {len(cycle_ids)} unique IDs" + + assert ( + completed_idx is not None + ), "response.completed event should be present" + + lite_errors = [ + record + for record in caplog.records + if record.levelno >= logging.ERROR + and ("LiteLLM" in record.name or "LiteLLM" in record.getMessage()) + ] + assert not lite_errors, "Unexpected LiteLLM errors: " + ", ".join( + record.getMessage() for record in lite_errors + ) diff --git a/tests/mcp_tests/test_mcp_litellm_client.py b/tests/mcp_tests/test_mcp_litellm_client.py index cfc0692c8fa..caabe18adb0 100644 --- a/tests/mcp_tests/test_mcp_litellm_client.py +++ b/tests/mcp_tests/test_mcp_litellm_client.py @@ -18,8 +18,8 @@ import json async def test_mcp_agent(): """Test MCP agent functionality with a simple math server""" try: - local_server_path = "./mcp_server.py" - ci_cd_server_path = "tests/mcp_tests/mcp_server.py" + local_server_path = "./mcp_math_server.py" + ci_cd_server_path = "tests/integration/mcp_sdk1/mcp_math_server.py" # Use the correct path for the server server_path = ( diff --git a/tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py b/tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py new file mode 100644 index 00000000000..23aa068ab5f --- /dev/null +++ b/tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py @@ -0,0 +1,53 @@ +import os + +import pytest + + +@pytest.mark.skipif( + not os.getenv("GEMINI_API_KEY") and not os.getenv("GOOGLE_API_KEY"), + reason="Requires GEMINI_API_KEY or GOOGLE_API_KEY.", +) +@pytest.mark.asyncio +async def test_gemini_pass_through_endpoint(): + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + Request, + Response, + gemini_proxy_route, + ) + + body = b""" + { + "contents": [{ + "parts":[{ + "text": "The quick brown fox jumps over the lazy dog." + }] + }] + } + """ + + # Construct the scope dictionary + scope = { + "type": "http", + "method": "POST", + "path": "/gemini/v1beta/models/gemini-2.5-flash:countTokens", + "query_string": b"key=sk-1234", + "headers": [ + (b"content-type", b"application/json"), + ], + } + + # Create a new Request object + async def async_receive(): + return {"type": "http.request", "body": body, "more_body": False} + + request = Request( + scope=scope, + receive=async_receive, + ) + + await gemini_proxy_route( + endpoint="v1beta/models/gemini-2.5-flash:countTokens?key=sk-1234", + request=request, + fastapi_response=Response(), + ) + diff --git a/tests/test_litellm/proxy/image_endpoints/test_azure_routes.py b/tests/test_litellm/proxy/image_endpoints/test_azure_routes.py index 91fff717d25..46fe9a6f893 100644 --- a/tests/test_litellm/proxy/image_endpoints/test_azure_routes.py +++ b/tests/test_litellm/proxy/image_endpoints/test_azure_routes.py @@ -53,7 +53,8 @@ def client_no_auth(): config_fp = ( repo_root / "tests" - / "proxy_unit_tests" + / "unit" + / "proxy" / "test_configs" / "test_config_no_auth.yaml" ) diff --git a/tests/test_litellm/test_assert_ci_coverage.py b/tests/test_litellm/test_assert_ci_coverage.py index 983707db606..8524a905745 100644 --- a/tests/test_litellm/test_assert_ci_coverage.py +++ b/tests/test_litellm/test_assert_ci_coverage.py @@ -340,7 +340,7 @@ def test_a_dockerfile_directory_entry_is_stale_because_only_an_exact_path_exempt def test_a_workflow_that_names_a_file_clears_it_from_the_slice_check(): named = coverage._workflow_named_tokens() assert named, "the workflows must name some test paths or the check proves nothing" - assert any(coverage._token_covers(token, "tests/local_testing/test_caching_handler.py") for token in named) + assert any(coverage._token_covers(token, "tests/proxy_unit_tests/test_proxy_custom_logger.py") for token in named) def test_the_slice_check_credits_only_workflows_never_the_circleci_config(): diff --git a/tests/test_litellm/test_circleci_path_filter.py b/tests/test_litellm/test_circleci_path_filter.py index 84e2327057d..dcce7f57113 100644 --- a/tests/test_litellm/test_circleci_path_filter.py +++ b/tests/test_litellm/test_circleci_path_filter.py @@ -107,7 +107,7 @@ CI = [".github/workflows/test-litellm-ui-unit.yml"] ), ( "cost-map-only", - ["model_prices_and_context_window.json", "tests/proxy_unit_tests/test_y.py"], + ["model_prices_and_context_window.json", "tests/unit/proxy/test_y.py"], "run", ), ( diff --git a/tests/proxy_unit_tests/test_key_generate_dynamodb.py b/tests/unit/caching/__init__.py similarity index 100% rename from tests/proxy_unit_tests/test_key_generate_dynamodb.py rename to tests/unit/caching/__init__.py diff --git a/tests/local_testing/test_cache_preset_key.py b/tests/unit/caching/test_cache_preset_key.py similarity index 100% rename from tests/local_testing/test_cache_preset_key.py rename to tests/unit/caching/test_cache_preset_key.py diff --git a/tests/local_testing/test_caching_handler.py b/tests/unit/caching/test_caching_handler.py similarity index 100% rename from tests/local_testing/test_caching_handler.py rename to tests/unit/caching/test_caching_handler.py diff --git a/tests/local_testing/test_responses_stream_cache_keys.py b/tests/unit/caching/test_responses_stream_cache_keys.py similarity index 100% rename from tests/local_testing/test_responses_stream_cache_keys.py rename to tests/unit/caching/test_responses_stream_cache_keys.py diff --git a/tests/local_testing/test_unit_test_caching.py b/tests/unit/caching/test_unit_test_caching.py similarity index 100% rename from tests/local_testing/test_unit_test_caching.py rename to tests/unit/caching/test_unit_test_caching.py diff --git a/tests/enterprise/conftest.py b/tests/unit/enterprise/conftest.py similarity index 100% rename from tests/enterprise/conftest.py rename to tests/unit/enterprise/conftest.py diff --git a/tests/unit/enterprise/enterprise_callbacks/send_emails/__init__.py b/tests/unit/enterprise/enterprise_callbacks/send_emails/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py b/tests/unit/enterprise/enterprise_callbacks/send_emails/test_base_email.py similarity index 100% rename from tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py rename to tests/unit/enterprise/enterprise_callbacks/send_emails/test_base_email.py diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_endpoints.py b/tests/unit/enterprise/enterprise_callbacks/send_emails/test_endpoints.py similarity index 100% rename from tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_endpoints.py rename to tests/unit/enterprise/enterprise_callbacks/send_emails/test_endpoints.py diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_resend_email.py b/tests/unit/enterprise/enterprise_callbacks/send_emails/test_resend_email.py similarity index 100% rename from tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_resend_email.py rename to tests/unit/enterprise/enterprise_callbacks/send_emails/test_resend_email.py diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py b/tests/unit/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py similarity index 100% rename from tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py rename to tests/unit/enterprise/enterprise_callbacks/send_emails/test_sendgrid_email.py diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py similarity index 100% rename from tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py rename to tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py diff --git a/tests/unit/enterprise/integrations/__init__.py b/tests/unit/enterprise/integrations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/enterprise/litellm_enterprise/integrations/test_custom_guardrail.py b/tests/unit/enterprise/integrations/test_custom_guardrail.py similarity index 100% rename from tests/enterprise/litellm_enterprise/integrations/test_custom_guardrail.py rename to tests/unit/enterprise/integrations/test_custom_guardrail.py diff --git a/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py b/tests/unit/enterprise/integrations/test_prometheus.py similarity index 100% rename from tests/enterprise/litellm_enterprise/integrations/test_prometheus.py rename to tests/unit/enterprise/integrations/test_prometheus.py diff --git a/tests/enterprise/litellm_enterprise/integrations/test_prometheus_unit_tests.py b/tests/unit/enterprise/integrations/test_prometheus_unit_tests.py similarity index 100% rename from tests/enterprise/litellm_enterprise/integrations/test_prometheus_unit_tests.py rename to tests/unit/enterprise/integrations/test_prometheus_unit_tests.py diff --git a/tests/unit/enterprise/proxy/__init__.py b/tests/unit/enterprise/proxy/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/enterprise/proxy/auth/__init__.py b/tests/unit/enterprise/proxy/auth/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/enterprise/litellm_enterprise/proxy/auth/test_route_checks.py b/tests/unit/enterprise/proxy/auth/test_route_checks.py similarity index 100% rename from tests/enterprise/litellm_enterprise/proxy/auth/test_route_checks.py rename to tests/unit/enterprise/proxy/auth/test_route_checks.py diff --git a/tests/enterprise/litellm_enterprise/proxy/auth/test_user_api_key_auth.py b/tests/unit/enterprise/proxy/auth/test_user_api_key_auth.py similarity index 100% rename from tests/enterprise/litellm_enterprise/proxy/auth/test_user_api_key_auth.py rename to tests/unit/enterprise/proxy/auth/test_user_api_key_auth.py diff --git a/tests/unit/enterprise/proxy/guardrails/__init__.py b/tests/unit/enterprise/proxy/guardrails/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/conftest.py b/tests/unit/enterprise/proxy/guardrails/conftest.py similarity index 100% rename from tests/enterprise/litellm_enterprise/proxy/guardrails/conftest.py rename to tests/unit/enterprise/proxy/guardrails/conftest.py diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py b/tests/unit/enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py similarity index 100% rename from tests/enterprise/litellm_enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py rename to tests/unit/enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py b/tests/unit/enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py similarity index 100% rename from tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py rename to tests/unit/enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py diff --git a/tests/unit/enterprise/proxy/hooks/__init__.py b/tests/unit/enterprise/proxy/hooks/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py b/tests/unit/enterprise/proxy/hooks/test_managed_files.py similarity index 100% rename from tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py rename to tests/unit/enterprise/proxy/hooks/test_managed_files.py diff --git a/tests/unit/enterprise/proxy/management_endpoints/__init__.py b/tests/unit/enterprise/proxy/management_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/enterprise/proxy/management_endpoints/test_internal_user_endpoints.py similarity index 100% rename from tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_internal_user_endpoints.py rename to tests/unit/enterprise/proxy/management_endpoints/test_internal_user_endpoints.py diff --git a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py b/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py similarity index 100% rename from tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py rename to tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py diff --git a/tests/test_litellm/enterprise/proxy/test_afile_retrieve_returns_unified_id.py b/tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py similarity index 100% rename from tests/test_litellm/enterprise/proxy/test_afile_retrieve_returns_unified_id.py rename to tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py diff --git a/tests/enterprise/litellm_enterprise/proxy/test_audit_logging_endpoints.py b/tests/unit/enterprise/proxy/test_audit_logging_endpoints.py similarity index 100% rename from tests/enterprise/litellm_enterprise/proxy/test_audit_logging_endpoints.py rename to tests/unit/enterprise/proxy/test_audit_logging_endpoints.py diff --git a/tests/test_litellm/enterprise/proxy/test_batch_retrieve_input_file_id.py b/tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py similarity index 100% rename from tests/test_litellm/enterprise/proxy/test_batch_retrieve_input_file_id.py rename to tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py diff --git a/tests/test_litellm/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py b/tests/unit/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py similarity index 100% rename from tests/test_litellm/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py rename to tests/unit/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py diff --git a/tests/test_litellm/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py b/tests/unit/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py similarity index 100% rename from tests/test_litellm/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py rename to tests/unit/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py diff --git a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py b/tests/unit/enterprise/proxy/test_batch_update_db_managed_output_file_id.py similarity index 100% rename from tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py rename to tests/unit/enterprise/proxy/test_batch_update_db_managed_output_file_id.py diff --git a/tests/test_litellm/enterprise/proxy/test_deleted_file_returns_403_not_404.py b/tests/unit/enterprise/proxy/test_deleted_file_returns_403_not_404.py similarity index 100% rename from tests/test_litellm/enterprise/proxy/test_deleted_file_returns_403_not_404.py rename to tests/unit/enterprise/proxy/test_deleted_file_returns_403_not_404.py diff --git a/tests/test_litellm/enterprise/proxy/test_enterprise_routes.py b/tests/unit/enterprise/proxy/test_enterprise_routes.py similarity index 100% rename from tests/test_litellm/enterprise/proxy/test_enterprise_routes.py rename to tests/unit/enterprise/proxy/test_enterprise_routes.py diff --git a/tests/test_litellm/enterprise/proxy/test_file_deletion_blocking.py b/tests/unit/enterprise/proxy/test_file_deletion_blocking.py similarity index 100% rename from tests/test_litellm/enterprise/proxy/test_file_deletion_blocking.py rename to tests/unit/enterprise/proxy/test_file_deletion_blocking.py diff --git a/tests/test_litellm/enterprise/proxy/test_managed_files_access_check.py b/tests/unit/enterprise/proxy/test_managed_files_access_check.py similarity index 100% rename from tests/test_litellm/enterprise/proxy/test_managed_files_access_check.py rename to tests/unit/enterprise/proxy/test_managed_files_access_check.py diff --git a/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py b/tests/unit/enterprise/proxy/test_managed_files_hook.py similarity index 100% rename from tests/test_litellm/enterprise/proxy/test_managed_files_hook.py rename to tests/unit/enterprise/proxy/test_managed_files_hook.py diff --git a/tests/unit/gateway/__init__.py b/tests/unit/gateway/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_gateway/test_launch.py b/tests/unit/gateway/test_launch.py similarity index 100% rename from tests/test_gateway/test_launch.py rename to tests/unit/gateway/test_launch.py diff --git a/tests/unit/litellm_proxy_extras/__init__.py b/tests/unit/litellm_proxy_extras/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_logging.py b/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_logging.py similarity index 100% rename from tests/litellm-proxy-extras/test_litellm_proxy_extras_logging.py rename to tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_logging.py diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py similarity index 99% rename from tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py rename to tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py index bb329264a11..755c7617701 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py @@ -9,7 +9,7 @@ import pytest sys.path.insert( 0, os.path.abspath( - os.path.join(os.path.dirname(__file__), "../../litellm-proxy-extras") + os.path.join(os.path.dirname(__file__), "../../../litellm-proxy-extras") ), ) @@ -23,7 +23,7 @@ from litellm_proxy_extras.utils import ( _MIGRATIONS_DIR = os.path.abspath( os.path.join( os.path.dirname(__file__), - "../../litellm-proxy-extras/litellm_proxy_extras/migrations", + "../../../litellm-proxy-extras/litellm_proxy_extras/migrations", ) ) @@ -999,7 +999,7 @@ class TestJWTKeyMappingCascade: schema_paths = glob.glob( os.path.abspath( os.path.join( - os.path.dirname(__file__), "../../**/schema.prisma" + os.path.dirname(__file__), "../../../**/schema.prisma" ) ), recursive=True, diff --git a/tests/unit/proxy/__init__.py b/tests/unit/proxy/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/_experimental/__init__.py b/tests/unit/proxy/_experimental/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/_experimental/mcp_server/__init__.py b/tests/unit/proxy/_experimental/mcp_server/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/_experimental/mcp_server/conftest.py b/tests/unit/proxy/_experimental/mcp_server/conftest.py new file mode 100644 index 00000000000..d8b91e07467 --- /dev/null +++ b/tests/unit/proxy/_experimental/mcp_server/conftest.py @@ -0,0 +1,78 @@ +import asyncio +import importlib + +import pytest + +import litellm +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + + +@pytest.fixture(scope="session") +def event_loop(): + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + yield loop + loop.close() + + +@pytest.fixture(scope="function", autouse=True) +def setup_and_teardown(): + """ + This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. + """ + importlib.reload(litellm) + import asyncio + + loop = asyncio.get_event_loop_policy().new_event_loop() + asyncio.set_event_loop(loop) + yield + + # Teardown code (executes after the yield point) + # LoggingWorker carries still-queued coroutines onto the next test's loop, where they'd log into that test's callbacks + asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue()) + loop.close() # Close the loop created earlier + asyncio.set_event_loop(None) # Remove the reference to the loop + + +@pytest.fixture(scope="function", autouse=True) +async def drain_logging_worker(): + """ + The logging queue is bound to the running loop, so anything left queued when a test's loop + goes away is carried onto the next test's loop and fires against its callbacks. + """ + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + + yield + + try: + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.clear_queue(), timeout=10) + except asyncio.TimeoutError: + pass + + +def pytest_collection_modifyitems(config, items): + # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests + custom_logger_tests = [ + item for item in items if "custom_logger" in item.parent.name + ] + other_tests = [item for item in items if "custom_logger" not in item.parent.name] + + # Sort tests based on their names + custom_logger_tests.sort(key=lambda x: x.name) + other_tests.sort(key=lambda x: x.name) + + # Reorder the items list + items[:] = custom_logger_tests + other_tests + + +@pytest.fixture +def config_only_mcp_manager_factory(): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + + class ConfigOnlyManager(MCPServerManager): + def initialize_tool_name_to_mcp_server_name_mapping(self): + return None + + return ConfigOnlyManager diff --git a/tests/mcp_tests/test_mcp_auth_header_extraction.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_auth_header_extraction.py similarity index 100% rename from tests/mcp_tests/test_mcp_auth_header_extraction.py rename to tests/unit/proxy/_experimental/mcp_server/test_mcp_auth_header_extraction.py diff --git a/tests/mcp_tests/test_mcp_auth_priority.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_auth_priority.py similarity index 100% rename from tests/mcp_tests/test_mcp_auth_priority.py rename to tests/unit/proxy/_experimental/mcp_server/test_mcp_auth_priority.py diff --git a/tests/mcp_tests/test_mcp_chat_completions.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_chat_completions.py similarity index 100% rename from tests/mcp_tests/test_mcp_chat_completions.py rename to tests/unit/proxy/_experimental/mcp_server/test_mcp_chat_completions.py diff --git a/tests/mcp_tests/test_mcp_client_unit.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_client_unit.py similarity index 100% rename from tests/mcp_tests/test_mcp_client_unit.py rename to tests/unit/proxy/_experimental/mcp_server/test_mcp_client_unit.py diff --git a/tests/mcp_tests/test_mcp_logging.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_logging.py similarity index 100% rename from tests/mcp_tests/test_mcp_logging.py rename to tests/unit/proxy/_experimental/mcp_server/test_mcp_logging.py diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py similarity index 100% rename from tests/mcp_tests/test_mcp_server.py rename to tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py diff --git a/tests/mcp_tests/test_oauth2_mcp_config.yaml b/tests/unit/proxy/_experimental/mcp_server/test_oauth2_mcp_config.yaml similarity index 100% rename from tests/mcp_tests/test_oauth2_mcp_config.yaml rename to tests/unit/proxy/_experimental/mcp_server/test_oauth2_mcp_config.yaml diff --git a/tests/mcp_tests/test_openapi_spec_path_url.py b/tests/unit/proxy/_experimental/mcp_server/test_openapi_spec_path_url.py similarity index 100% rename from tests/mcp_tests/test_openapi_spec_path_url.py rename to tests/unit/proxy/_experimental/mcp_server/test_openapi_spec_path_url.py diff --git a/tests/mcp_tests/test_per_user_oauth_cache.py b/tests/unit/proxy/_experimental/mcp_server/test_per_user_oauth_cache.py similarity index 100% rename from tests/mcp_tests/test_per_user_oauth_cache.py rename to tests/unit/proxy/_experimental/mcp_server/test_per_user_oauth_cache.py diff --git a/tests/unit/proxy/auth/__init__.py b/tests/unit/proxy/auth/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/unit/proxy/auth/test_auth_checks.py similarity index 100% rename from tests/proxy_unit_tests/test_auth_checks.py rename to tests/unit/proxy/auth/test_auth_checks.py diff --git a/tests/proxy_unit_tests/test_default_end_user_budget_simple.py b/tests/unit/proxy/auth/test_default_end_user_budget_simple.py similarity index 100% rename from tests/proxy_unit_tests/test_default_end_user_budget_simple.py rename to tests/unit/proxy/auth/test_default_end_user_budget_simple.py diff --git a/tests/proxy_unit_tests/test_jwt.py b/tests/unit/proxy/auth/test_jwt.py similarity index 100% rename from tests/proxy_unit_tests/test_jwt.py rename to tests/unit/proxy/auth/test_jwt.py diff --git a/tests/proxy_unit_tests/test_models_fallback_endpoint.py b/tests/unit/proxy/auth/test_models_fallback_endpoint.py similarity index 100% rename from tests/proxy_unit_tests/test_models_fallback_endpoint.py rename to tests/unit/proxy/auth/test_models_fallback_endpoint.py diff --git a/tests/proxy_unit_tests/test_multipart_bypass_repro.py b/tests/unit/proxy/auth/test_multipart_bypass_repro.py similarity index 100% rename from tests/proxy_unit_tests/test_multipart_bypass_repro.py rename to tests/unit/proxy/auth/test_multipart_bypass_repro.py diff --git a/tests/proxy_unit_tests/test_proxy_routes.py b/tests/unit/proxy/auth/test_proxy_routes.py similarity index 100% rename from tests/proxy_unit_tests/test_proxy_routes.py rename to tests/unit/proxy/auth/test_proxy_routes.py diff --git a/tests/proxy_unit_tests/test_user_api_key_auth.py b/tests/unit/proxy/auth/test_user_api_key_auth.py similarity index 100% rename from tests/proxy_unit_tests/test_user_api_key_auth.py rename to tests/unit/proxy/auth/test_user_api_key_auth.py diff --git a/tests/unit/proxy/common_utils/__init__.py b/tests/unit/proxy/common_utils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py similarity index 100% rename from tests/proxy_unit_tests/test_check_batch_cost.py rename to tests/unit/proxy/common_utils/test_check_batch_cost.py diff --git a/tests/proxy_unit_tests/test_check_responses_cost.py b/tests/unit/proxy/common_utils/test_check_responses_cost.py similarity index 100% rename from tests/proxy_unit_tests/test_check_responses_cost.py rename to tests/unit/proxy/common_utils/test_check_responses_cost.py diff --git a/tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py b/tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py similarity index 100% rename from tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py rename to tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py diff --git a/tests/proxy_unit_tests/test_realtime_cache.py b/tests/unit/proxy/common_utils/test_realtime_cache.py similarity index 100% rename from tests/proxy_unit_tests/test_realtime_cache.py rename to tests/unit/proxy/common_utils/test_realtime_cache.py diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py new file mode 100644 index 00000000000..148751c33f2 --- /dev/null +++ b/tests/unit/proxy/conftest.py @@ -0,0 +1,150 @@ +# conftest.py + +import asyncio +import copy +import inspect +import warnings + +import pytest + + +import litellm +import litellm.proxy.proxy_server + + +# Top-level assignments of these types are the ones importlib.reload(litellm) +# would have effectively reset. We snapshot them at conftest import time and +# deep-copy the snapshot back before every test. +_SNAPSHOT_TYPES = (list, dict, set, tuple, str, int, float, bool, bytes) + + +def _snapshot_mutable_state(module): + """Capture a per-module snapshot of primitive and collection attributes.""" + snapshot = {} + for attr in list(vars(module)): + if attr.startswith("_"): + continue + try: + value = getattr(module, attr) + except Exception as exc: + warnings.warn( + f"conftest: could not read {module.__name__}.{attr} during snapshot: {exc}", + stacklevel=2, + ) + continue + if value is None or isinstance(value, _SNAPSHOT_TYPES): + try: + snapshot[attr] = copy.deepcopy(value) + except Exception as exc: + warnings.warn( + f"conftest: could not snapshot {module.__name__}.{attr}: {exc}", + stacklevel=2, + ) + return snapshot + + +def _restore_mutable_state(module, snapshot): + for attr, default in snapshot.items(): + try: + setattr(module, attr, copy.deepcopy(default)) + except Exception as exc: + warnings.warn( + f"conftest: could not restore {module.__name__}.{attr}: {exc}", + stacklevel=2, + ) + + +def _collect_flushable_caches(): + """Return (module, attr) pairs whose values expose flush_cache().""" + targets = [] + for module in (litellm, litellm.proxy.proxy_server): + for attr in list(vars(module)): + if attr.startswith("_"): + continue + try: + value = getattr(module, attr) + except Exception: + continue + # Only instances โ€” a class reference has an unbound flush_cache + # that can't be called without a self argument. + if inspect.isclass(value) or inspect.ismodule(value): + continue + if callable(getattr(value, "flush_cache", None)): + targets.append((module, attr)) + return targets + + +def _flush_caches(targets): + for module, attr in targets: + try: + value = getattr(module, attr) + except Exception: + continue + flush = getattr(value, "flush_cache", None) + if callable(flush): + try: + flush() + except Exception as exc: + warnings.warn( + f"conftest: flush_cache failed on {module.__name__}.{attr}: {exc}", + stacklevel=2, + ) + + +# Snapshot once at conftest import โ€” these are the "clean" module states. +_LITELLM_STATE = _snapshot_mutable_state(litellm) +_PROXY_SERVER_STATE = _snapshot_mutable_state(litellm.proxy.proxy_server) +_FLUSHABLE_CACHES = _collect_flushable_caches() + + +@pytest.fixture(scope="function", autouse=True) +def setup_and_teardown(): + """Reset mutable module state on litellm and proxy_server before each test. + + Replaces a previous importlib.reload(litellm) approach that cost ~17s + per test (re-executing the full litellm __init__ import chain). + + What IS reset: + - Top-level module attributes of type list / dict / set / tuple + / str / int / float / bool / bytes, and None-valued attributes. + These cover callback lists, general_settings, master_key, + premium_user, prisma_client, etc. โ€” anything the old reload() reset + by re-executing the module body. + - Any module-level object instance that exposes flush_cache() (the + DualCache and LLMClientCache family), which handles cache state + that can't round-trip through deepcopy because of internal locks. + + What is NOT reset: + - Class instances without flush_cache() (e.g. ProxyLogging, + JWTHandler, FastAPI routers, loggers). If a test mutates such an + instance in-place (setattr on the instance, appending to one of + its internal lists, etc.), the mutation will leak into later tests. + Use pytest's monkeypatch.setattr() or a local fixture for those + cases โ€” don't rely on this autouse fixture to undo them. + """ + _restore_mutable_state(litellm, _LITELLM_STATE) + _restore_mutable_state(litellm.proxy.proxy_server, _PROXY_SERVER_STATE) + _flush_caches(_FLUSHABLE_CACHES) + + loop = asyncio.get_event_loop_policy().new_event_loop() + asyncio.set_event_loop(loop) + try: + yield + finally: + loop.close() + asyncio.set_event_loop(None) + + +def pytest_collection_modifyitems(config, items): + # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests + custom_logger_tests = [ + item for item in items if "custom_logger" in item.parent.name + ] + other_tests = [item for item in items if "custom_logger" not in item.parent.name] + + # Sort tests based on their names + custom_logger_tests.sort(key=lambda x: x.name) + other_tests.sort(key=lambda x: x.name) + + # Reorder the items list + items[:] = custom_logger_tests + other_tests diff --git a/tests/unit/proxy/db/__init__.py b/tests/unit/proxy/db/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/proxy/db/db_transaction_queue/__init__.py b/tests/unit/proxy/db/db_transaction_queue/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/proxy_unit_tests/test_e2e_pod_lock_manager.py b/tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py similarity index 100% rename from tests/proxy_unit_tests/test_e2e_pod_lock_manager.py rename to tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py diff --git a/tests/proxy_unit_tests/test_update_daily_tag_spend.py b/tests/unit/proxy/db/test_update_daily_tag_spend.py similarity index 100% rename from tests/proxy_unit_tests/test_update_daily_tag_spend.py rename to tests/unit/proxy/db/test_update_daily_tag_spend.py diff --git a/tests/proxy_unit_tests/example_config_yaml/aliases_config.yaml b/tests/unit/proxy/example_config_yaml/aliases_config.yaml similarity index 100% rename from tests/proxy_unit_tests/example_config_yaml/aliases_config.yaml rename to tests/unit/proxy/example_config_yaml/aliases_config.yaml diff --git a/tests/proxy_unit_tests/example_config_yaml/azure_config.yaml b/tests/unit/proxy/example_config_yaml/azure_config.yaml similarity index 100% rename from tests/proxy_unit_tests/example_config_yaml/azure_config.yaml rename to tests/unit/proxy/example_config_yaml/azure_config.yaml diff --git a/tests/proxy_unit_tests/example_config_yaml/cache_no_params.yaml b/tests/unit/proxy/example_config_yaml/cache_no_params.yaml similarity index 100% rename from tests/proxy_unit_tests/example_config_yaml/cache_no_params.yaml rename to tests/unit/proxy/example_config_yaml/cache_no_params.yaml diff --git a/tests/proxy_unit_tests/example_config_yaml/cache_with_params.yaml b/tests/unit/proxy/example_config_yaml/cache_with_params.yaml similarity index 100% rename from tests/proxy_unit_tests/example_config_yaml/cache_with_params.yaml rename to tests/unit/proxy/example_config_yaml/cache_with_params.yaml diff --git a/tests/proxy_unit_tests/example_config_yaml/config_with_env_vars.yaml b/tests/unit/proxy/example_config_yaml/config_with_env_vars.yaml similarity index 100% rename from tests/proxy_unit_tests/example_config_yaml/config_with_env_vars.yaml rename to tests/unit/proxy/example_config_yaml/config_with_env_vars.yaml diff --git a/tests/proxy_unit_tests/example_config_yaml/config_with_include.yaml b/tests/unit/proxy/example_config_yaml/config_with_include.yaml similarity index 100% rename from tests/proxy_unit_tests/example_config_yaml/config_with_include.yaml rename to tests/unit/proxy/example_config_yaml/config_with_include.yaml diff --git a/tests/proxy_unit_tests/example_config_yaml/config_with_missing_include.yaml b/tests/unit/proxy/example_config_yaml/config_with_missing_include.yaml similarity index 100% rename from tests/proxy_unit_tests/example_config_yaml/config_with_missing_include.yaml rename to tests/unit/proxy/example_config_yaml/config_with_missing_include.yaml diff --git a/tests/proxy_unit_tests/example_config_yaml/config_with_multiple_includes.yaml b/tests/unit/proxy/example_config_yaml/config_with_multiple_includes.yaml similarity index 100% rename from tests/proxy_unit_tests/example_config_yaml/config_with_multiple_includes.yaml rename to tests/unit/proxy/example_config_yaml/config_with_multiple_includes.yaml diff --git a/tests/proxy_unit_tests/example_config_yaml/included_models.yaml b/tests/unit/proxy/example_config_yaml/included_models.yaml similarity index 100% rename from tests/proxy_unit_tests/example_config_yaml/included_models.yaml rename to tests/unit/proxy/example_config_yaml/included_models.yaml diff --git a/tests/proxy_unit_tests/example_config_yaml/langfuse_config.yaml b/tests/unit/proxy/example_config_yaml/langfuse_config.yaml similarity index 100% rename from tests/proxy_unit_tests/example_config_yaml/langfuse_config.yaml rename to tests/unit/proxy/example_config_yaml/langfuse_config.yaml diff --git a/tests/proxy_unit_tests/example_config_yaml/load_balancer.yaml b/tests/unit/proxy/example_config_yaml/load_balancer.yaml similarity index 100% rename from tests/proxy_unit_tests/example_config_yaml/load_balancer.yaml rename to tests/unit/proxy/example_config_yaml/load_balancer.yaml diff --git a/tests/proxy_unit_tests/example_config_yaml/models_file_1.yaml b/tests/unit/proxy/example_config_yaml/models_file_1.yaml similarity index 100% rename from tests/proxy_unit_tests/example_config_yaml/models_file_1.yaml rename to tests/unit/proxy/example_config_yaml/models_file_1.yaml diff --git a/tests/proxy_unit_tests/example_config_yaml/models_file_2.yaml b/tests/unit/proxy/example_config_yaml/models_file_2.yaml similarity index 100% rename from tests/proxy_unit_tests/example_config_yaml/models_file_2.yaml rename to tests/unit/proxy/example_config_yaml/models_file_2.yaml diff --git a/tests/proxy_unit_tests/example_config_yaml/opentelemetry_config.yaml b/tests/unit/proxy/example_config_yaml/opentelemetry_config.yaml similarity index 100% rename from tests/proxy_unit_tests/example_config_yaml/opentelemetry_config.yaml rename to tests/unit/proxy/example_config_yaml/opentelemetry_config.yaml diff --git a/tests/proxy_unit_tests/example_config_yaml/simple_config.yaml b/tests/unit/proxy/example_config_yaml/simple_config.yaml similarity index 100% rename from tests/proxy_unit_tests/example_config_yaml/simple_config.yaml rename to tests/unit/proxy/example_config_yaml/simple_config.yaml diff --git a/tests/unit/proxy/google_endpoints/__init__.py b/tests/unit/proxy/google_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/proxy_unit_tests/test_gemini_agents_endpoints.py b/tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py similarity index 100% rename from tests/proxy_unit_tests/test_gemini_agents_endpoints.py rename to tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py diff --git a/tests/proxy_unit_tests/test_google_endpoint_routing.py b/tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py similarity index 100% rename from tests/proxy_unit_tests/test_google_endpoint_routing.py rename to tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py diff --git a/tests/proxy_unit_tests/test_google_gemini_proxy_request.py b/tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py similarity index 100% rename from tests/proxy_unit_tests/test_google_gemini_proxy_request.py rename to tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py diff --git a/tests/unit/proxy/hooks/__init__.py b/tests/unit/proxy/hooks/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/proxy_unit_tests/test_banned_keyword_list.py b/tests/unit/proxy/hooks/test_banned_keyword_list.py similarity index 100% rename from tests/proxy_unit_tests/test_banned_keyword_list.py rename to tests/unit/proxy/hooks/test_banned_keyword_list.py diff --git a/tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py b/tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py similarity index 100% rename from tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py rename to tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py diff --git a/tests/unit/proxy/management_endpoints/__init__.py b/tests/unit/proxy/management_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/proxy_unit_tests/test_jwt_key_mapping.py b/tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py similarity index 99% rename from tests/proxy_unit_tests/test_jwt_key_mapping.py rename to tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py index e95ed42013b..50b7a5c03fd 100644 --- a/tests/proxy_unit_tests/test_jwt_key_mapping.py +++ b/tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py @@ -5,7 +5,7 @@ from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch # Add project root to sys.path -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../.."))) from litellm.proxy.auth.user_api_key_auth import ( _resolve_jwt_to_virtual_key, diff --git a/tests/proxy_unit_tests/test_key_generate_prisma.py b/tests/unit/proxy/management_endpoints/test_key_generate_prisma.py similarity index 100% rename from tests/proxy_unit_tests/test_key_generate_prisma.py rename to tests/unit/proxy/management_endpoints/test_key_generate_prisma.py diff --git a/tests/unit/proxy/management_helpers/__init__.py b/tests/unit/proxy/management_helpers/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/proxy_unit_tests/test_audit_logs_proxy.py b/tests/unit/proxy/management_helpers/test_audit_logs_proxy.py similarity index 100% rename from tests/proxy_unit_tests/test_audit_logs_proxy.py rename to tests/unit/proxy/management_helpers/test_audit_logs_proxy.py diff --git a/tests/unit/proxy/middleware/__init__.py b/tests/unit/proxy/middleware/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/proxy_unit_tests/test_request_size_limit_middleware.py b/tests/unit/proxy/middleware/test_request_size_limit_middleware.py similarity index 100% rename from tests/proxy_unit_tests/test_request_size_limit_middleware.py rename to tests/unit/proxy/middleware/test_request_size_limit_middleware.py diff --git a/tests/unit/proxy/public_endpoints/__init__.py b/tests/unit/proxy/public_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/proxy_unit_tests/test_blog_posts_endpoint.py b/tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py similarity index 100% rename from tests/proxy_unit_tests/test_blog_posts_endpoint.py rename to tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py diff --git a/tests/unit/proxy/response_polling/__init__.py b/tests/unit/proxy/response_polling/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/proxy_unit_tests/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py similarity index 100% rename from tests/proxy_unit_tests/test_response_polling_handler.py rename to tests/unit/proxy/response_polling/test_response_polling_handler.py diff --git a/tests/unit/proxy/spend_tracking/__init__.py b/tests/unit/proxy/spend_tracking/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/proxy_unit_tests/test_search_api_logging.py b/tests/unit/proxy/spend_tracking/test_search_api_logging.py similarity index 100% rename from tests/proxy_unit_tests/test_search_api_logging.py rename to tests/unit/proxy/spend_tracking/test_search_api_logging.py diff --git a/tests/proxy_unit_tests/test_aproxy_startup.py b/tests/unit/proxy/test_aproxy_startup.py similarity index 100% rename from tests/proxy_unit_tests/test_aproxy_startup.py rename to tests/unit/proxy/test_aproxy_startup.py diff --git a/tests/proxy_unit_tests/test_configs/custom_auth.py b/tests/unit/proxy/test_configs/custom_auth.py similarity index 100% rename from tests/proxy_unit_tests/test_configs/custom_auth.py rename to tests/unit/proxy/test_configs/custom_auth.py diff --git a/tests/proxy_unit_tests/test_configs/test_cloudflare_azure_with_cache_config.yaml b/tests/unit/proxy/test_configs/test_cloudflare_azure_with_cache_config.yaml similarity index 100% rename from tests/proxy_unit_tests/test_configs/test_cloudflare_azure_with_cache_config.yaml rename to tests/unit/proxy/test_configs/test_cloudflare_azure_with_cache_config.yaml diff --git a/tests/proxy_unit_tests/test_configs/test_config.yaml b/tests/unit/proxy/test_configs/test_config.yaml similarity index 100% rename from tests/proxy_unit_tests/test_configs/test_config.yaml rename to tests/unit/proxy/test_configs/test_config.yaml diff --git a/tests/proxy_unit_tests/test_configs/test_config_custom_auth.yaml b/tests/unit/proxy/test_configs/test_config_custom_auth.yaml similarity index 100% rename from tests/proxy_unit_tests/test_configs/test_config_custom_auth.yaml rename to tests/unit/proxy/test_configs/test_config_custom_auth.yaml diff --git a/tests/proxy_unit_tests/test_configs/test_config_no_auth.yaml b/tests/unit/proxy/test_configs/test_config_no_auth.yaml similarity index 100% rename from tests/proxy_unit_tests/test_configs/test_config_no_auth.yaml rename to tests/unit/proxy/test_configs/test_config_no_auth.yaml diff --git a/tests/proxy_unit_tests/test_configs/test_guardrails_config.yaml b/tests/unit/proxy/test_configs/test_guardrails_config.yaml similarity index 100% rename from tests/proxy_unit_tests/test_configs/test_guardrails_config.yaml rename to tests/unit/proxy/test_configs/test_guardrails_config.yaml diff --git a/tests/proxy_unit_tests/test_custom_callback_input.py b/tests/unit/proxy/test_custom_callback_input.py similarity index 100% rename from tests/proxy_unit_tests/test_custom_callback_input.py rename to tests/unit/proxy/test_custom_callback_input.py diff --git a/tests/proxy_unit_tests/test_custom_logger_s3_gcs.py b/tests/unit/proxy/test_custom_logger_s3_gcs.py similarity index 100% rename from tests/proxy_unit_tests/test_custom_logger_s3_gcs.py rename to tests/unit/proxy/test_custom_logger_s3_gcs.py diff --git a/tests/proxy_unit_tests/test_custom_tokenizer_bug.py b/tests/unit/proxy/test_custom_tokenizer_bug.py similarity index 100% rename from tests/proxy_unit_tests/test_custom_tokenizer_bug.py rename to tests/unit/proxy/test_custom_tokenizer_bug.py diff --git a/tests/proxy_unit_tests/test_db_schema_changes.py b/tests/unit/proxy/test_db_schema_changes.py similarity index 100% rename from tests/proxy_unit_tests/test_db_schema_changes.py rename to tests/unit/proxy/test_db_schema_changes.py diff --git a/tests/proxy_unit_tests/test_deprecated_key_grace_period.py b/tests/unit/proxy/test_deprecated_key_grace_period.py similarity index 100% rename from tests/proxy_unit_tests/test_deprecated_key_grace_period.py rename to tests/unit/proxy/test_deprecated_key_grace_period.py diff --git a/tests/proxy_unit_tests/test_get_favicon.py b/tests/unit/proxy/test_get_favicon.py similarity index 100% rename from tests/proxy_unit_tests/test_get_favicon.py rename to tests/unit/proxy/test_get_favicon.py diff --git a/tests/proxy_unit_tests/test_get_image.py b/tests/unit/proxy/test_get_image.py similarity index 100% rename from tests/proxy_unit_tests/test_get_image.py rename to tests/unit/proxy/test_get_image.py diff --git a/tests/proxy_unit_tests/test_prisma_client_backoff_retry.py b/tests/unit/proxy/test_prisma_client_backoff_retry.py similarity index 100% rename from tests/proxy_unit_tests/test_prisma_client_backoff_retry.py rename to tests/unit/proxy/test_prisma_client_backoff_retry.py diff --git a/tests/proxy_unit_tests/test_prompt_test_endpoint.py b/tests/unit/proxy/test_prompt_test_endpoint.py similarity index 100% rename from tests/proxy_unit_tests/test_prompt_test_endpoint.py rename to tests/unit/proxy/test_prompt_test_endpoint.py diff --git a/tests/proxy_unit_tests/test_proxy_config_unit_test.py b/tests/unit/proxy/test_proxy_config_unit_test.py similarity index 100% rename from tests/proxy_unit_tests/test_proxy_config_unit_test.py rename to tests/unit/proxy/test_proxy_config_unit_test.py diff --git a/tests/proxy_unit_tests/test_proxy_custom_auth.py b/tests/unit/proxy/test_proxy_custom_auth.py similarity index 100% rename from tests/proxy_unit_tests/test_proxy_custom_auth.py rename to tests/unit/proxy/test_proxy_custom_auth.py diff --git a/tests/proxy_unit_tests/test_proxy_pass_user_config.py b/tests/unit/proxy/test_proxy_pass_user_config.py similarity index 100% rename from tests/proxy_unit_tests/test_proxy_pass_user_config.py rename to tests/unit/proxy/test_proxy_pass_user_config.py diff --git a/tests/proxy_unit_tests/test_proxy_reject_logging.py b/tests/unit/proxy/test_proxy_reject_logging.py similarity index 100% rename from tests/proxy_unit_tests/test_proxy_reject_logging.py rename to tests/unit/proxy/test_proxy_reject_logging.py diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/unit/proxy/test_proxy_server.py similarity index 98% rename from tests/proxy_unit_tests/test_proxy_server.py rename to tests/unit/proxy/test_proxy_server.py index ed0380058a5..eae80f311d8 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/unit/proxy/test_proxy_server.py @@ -477,7 +477,7 @@ async def test_team_disable_guardrails(mock_acompletion, client_no_auth): assert e.code == str(403) -from test_custom_callback_input import CompletionCustomHandler +from tests.unit.proxy.test_custom_callback_input import CompletionCustomHandler @mock_patch_acompletion() @@ -1114,7 +1114,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.management_endpoints.internal_user_endpoints import new_user from litellm.proxy.management_endpoints.team_endpoints import team_member_add -from test_key_generate_prisma import prisma_client +from tests.unit.proxy.management_endpoints.test_key_generate_prisma import prisma_client @pytest.fixture @@ -2065,59 +2065,6 @@ async def test_add_callback_via_key_litellm_pre_call_utils_langsmith( assert new_data["failure_callback"] == expected_failure_callbacks -@pytest.mark.skipif( - not os.getenv("GEMINI_API_KEY") and not os.getenv("GOOGLE_API_KEY"), - reason="Requires GEMINI_API_KEY or GOOGLE_API_KEY.", -) -@pytest.mark.asyncio -async def test_gemini_pass_through_endpoint(): - from starlette.datastructures import URL - - from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( - Request, - Response, - gemini_proxy_route, - ) - - body = b""" - { - "contents": [{ - "parts":[{ - "text": "The quick brown fox jumps over the lazy dog." - }] - }] - } - """ - - # Construct the scope dictionary - scope = { - "type": "http", - "method": "POST", - "path": "/gemini/v1beta/models/gemini-2.5-flash:countTokens", - "query_string": b"key=sk-1234", - "headers": [ - (b"content-type", b"application/json"), - ], - } - - # Create a new Request object - async def async_receive(): - return {"type": "http.request", "body": body, "more_body": False} - - request = Request( - scope=scope, - receive=async_receive, - ) - - resp = await gemini_proxy_route( - endpoint="v1beta/models/gemini-2.5-flash:countTokens?key=sk-1234", - request=request, - fastapi_response=Response(), - ) - - print(resp.body) - - @pytest.mark.parametrize("hidden", [True, False]) @pytest.mark.asyncio async def test_model_info_alias_without_prisma(hidden): diff --git a/tests/proxy_unit_tests/test_proxy_setting_guardrails.py b/tests/unit/proxy/test_proxy_setting_guardrails.py similarity index 100% rename from tests/proxy_unit_tests/test_proxy_setting_guardrails.py rename to tests/unit/proxy/test_proxy_setting_guardrails.py diff --git a/tests/proxy_unit_tests/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py similarity index 100% rename from tests/proxy_unit_tests/test_proxy_token_counter.py rename to tests/unit/proxy/test_proxy_token_counter.py diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/unit/proxy/test_proxy_utils.py similarity index 100% rename from tests/proxy_unit_tests/test_proxy_utils.py rename to tests/unit/proxy/test_proxy_utils.py diff --git a/tests/proxy_unit_tests/test_reducto_ocr_route.py b/tests/unit/proxy/test_reducto_ocr_route.py similarity index 100% rename from tests/proxy_unit_tests/test_reducto_ocr_route.py rename to tests/unit/proxy/test_reducto_ocr_route.py diff --git a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py b/tests/unit/proxy/test_response_polling_pre_call_checks.py similarity index 100% rename from tests/proxy_unit_tests/test_response_polling_pre_call_checks.py rename to tests/unit/proxy/test_response_polling_pre_call_checks.py diff --git a/tests/proxy_unit_tests/test_server_root_path.py b/tests/unit/proxy/test_server_root_path.py similarity index 100% rename from tests/proxy_unit_tests/test_server_root_path.py rename to tests/unit/proxy/test_server_root_path.py diff --git a/tests/proxy_unit_tests/test_ui_path_detection.py b/tests/unit/proxy/test_ui_path_detection.py similarity index 100% rename from tests/proxy_unit_tests/test_ui_path_detection.py rename to tests/unit/proxy/test_ui_path_detection.py diff --git a/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py b/tests/unit/proxy/test_unit_test_proxy_hooks.py similarity index 100% rename from tests/proxy_unit_tests/test_unit_test_proxy_hooks.py rename to tests/unit/proxy/test_unit_test_proxy_hooks.py diff --git a/tests/proxy_unit_tests/test_update_spend.py b/tests/unit/proxy/test_update_spend.py similarity index 100% rename from tests/proxy_unit_tests/test_update_spend.py rename to tests/unit/proxy/test_update_spend.py diff --git a/tests/proxy_unit_tests/test_zero_cost_model_budget_bypass.py b/tests/unit/proxy/test_zero_cost_model_budget_bypass.py similarity index 100% rename from tests/proxy_unit_tests/test_zero_cost_model_budget_bypass.py rename to tests/unit/proxy/test_zero_cost_model_budget_bypass.py diff --git a/tests/proxy_unit_tests/vertex_key.json b/tests/unit/proxy/vertex_key.json similarity index 100% rename from tests/proxy_unit_tests/vertex_key.json rename to tests/unit/proxy/vertex_key.json diff --git a/tests/unit/responses/__init__.py b/tests/unit/responses/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/responses/mcp/__init__.py b/tests/unit/responses/mcp/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/mcp_tests/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py similarity index 77% rename from tests/mcp_tests/test_aresponses_api_with_mcp.py rename to tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index eb6f78b57a1..de0dc78af43 100644 --- a/tests/mcp_tests/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -1,5 +1,3 @@ -import logging -import os import pytest from mcp.types import Tool as MCPTool from typing import List, Any, cast @@ -846,161 +844,6 @@ async def test_streaming_mcp_events_validation(): assert mock_get_tools.called, "MCP tools should have been fetched" -@pytest.mark.asyncio -@pytest.mark.parametrize( - "model", - [ - pytest.param("gpt-4o-mini", id="openai"), - pytest.param("claude-haiku-4-5", id="anthropic"), - ], -) -async def test_streaming_responses_api_with_mcp_tools( - model: str, caplog: pytest.LogCaptureFixture -): - """ - Test the streaming responses API with MCP tools when using server_url="litellm_proxy" - - Under the hood the follow occurs - - - MCP: responses called litellm MCP manager.list_tools (MOCKED) - - Request 1: Made to model under test with fetched tools (REAL LLM CALL) - - MCP: Execute tool call from request 1 and returns result (MOCKED) - - Request 2: Made to model under test with fetched tools and tool results (REAL LLM CALL) - - Return the user the result of request 2 - """ - # Skip test if API keys are not set for the respective models - if ("claude" in model.lower() or "anthropic" in model.lower()) and not os.getenv( - "ANTHROPIC_API_KEY" - ): - pytest.skip("ANTHROPIC_API_KEY not set, skipping anthropic model test") - if ("gpt" in model.lower() or "openai" in model.lower()) and not os.getenv( - "OPENAI_API_KEY" - ): - pytest.skip("OPENAI_API_KEY not set, skipping openai model test") - - from unittest.mock import AsyncMock, patch - - print("๐Ÿงช Testing basic streaming with MCP tools...") - - # Mock MCP tools that would be returned from the manager - mock_mcp_tools = [ - MCPTool.model_validate({ - "name": "search_repo", - "description": "Search BerriAI/litellm repository for information", - "inputSchema": { - "type": "object", - "properties": { - "query": {"type": "string", "description": "Search query"} - }, - "required": ["query"], - }, - }, by_name=False) - ] - - # Only mock the MCP-specific operations, let LLM responses be real - with caplog.at_level(logging.ERROR): - with ( - patch.object( - LiteLLM_Proxy_MCP_Handler, - "_get_mcp_tools_from_manager", - new_callable=AsyncMock, - ) as mock_get_tools, - patch.object( - LiteLLM_Proxy_MCP_Handler, - "_execute_tool_calls", - new_callable=AsyncMock, - ) as mock_execute_tools, - ): - # Setup MCP mocks only - mock_get_tools.return_value = (mock_mcp_tools, ["litellm_proxy"]) - - # Create a dynamic mock that will match the actual tool call ID from the LLM response - def mock_execute_tool_calls_side_effect( - tool_calls, user_api_key_auth, **kwargs - ): - """Mock function that returns results matching the actual tool call IDs from the LLM""" - results = [] - for tool_call in tool_calls: - # Extract call_id from the tool call - call_id = None - if isinstance(tool_call, dict): - call_id = tool_call.get("call_id") or tool_call.get("id") - elif hasattr(tool_call, "call_id"): - call_id = tool_call.call_id - elif hasattr(tool_call, "id"): - call_id = tool_call.id - - if call_id: - results.append( - { - "tool_call_id": call_id, - "result": "LiteLLM is a unified interface for 100+ LLMs that translates inputs to provider-specific completion endpoints and provides consistent OpenAI-format output.", - } - ) - return results - - mock_execute_tools.side_effect = mock_execute_tool_calls_side_effect - - # Make the actual call - LLM responses will be real - mcp_tool_config = cast( - Any, - { - "type": "mcp", - "server_url": "litellm_proxy", - "require_approval": "never", - }, - ) - response = await litellm.aresponses( - model=model, - tools=[mcp_tool_config], - tool_choice="required", - input=[ - { - "role": "user", - "type": "message", - "content": "give me a TLDR of what BerriAI/litellm is about", - } - ], - stream=True, - ) - - print(f"๐Ÿ“‹ Response type: {type(response)}") - assert hasattr( - response, "__aiter__" - ), "Response should be an async streaming response" - - # Collect streaming chunks - chunks = [] - async for chunk in response: - chunks.append(chunk) - print(f"๐Ÿ“ฆ Chunk type: {getattr(chunk, 'type', 'unknown')}") - - print(f"๐Ÿ“Š Total chunks received: {len(chunks)}") - - # Verify MCP mocks were called (may be called multiple times in streaming) - assert ( - mock_get_tools.call_count >= 1 - ), f"Expected MCP tools to be fetched at least once, got {mock_get_tools.call_count}" - print(f"MCP tools fetched: {len(mock_mcp_tools)}") - - # Verify we got a response - assert response is not None - assert len(chunks) > 0, "Should have received streaming chunks" - - print("Basic streaming responses API with MCP tools test passed!") - - lite_errors = [ - record - for record in caplog.records - if record.levelno >= logging.ERROR - and ("LiteLLM" in record.name or "LiteLLM" in record.getMessage()) - ] - assert not lite_errors, "Unexpected LiteLLM errors: " + ", ".join( - record.getMessage() for record in lite_errors - ) - - @pytest.mark.asyncio async def test_mcp_parameter_preparation_helpers(): """ @@ -1215,7 +1058,7 @@ async def test_no_duplicate_mcp_tools_in_streaming_e2e(): The test mocks the MCP manager response but validates the actual tools sent to the LLM to ensure no duplication occurs. """ - from unittest.mock import AsyncMock, patch, call + from unittest.mock import AsyncMock, patch from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) @@ -1432,221 +1275,3 @@ async def test_no_duplicate_mcp_tools_in_streaming_e2e(): "tools_per_call": [len(tools) for tools in llm_call_tools], "duplicate_tools_found": False, } - - -@pytest.mark.asyncio -@pytest.mark.parametrize("model", ["gpt-4o-mini"]) -async def test_streaming_mcp_event_order_and_response_id_consistency( - model: str, caplog: pytest.LogCaptureFixture -): - """ - Test that: - 1. Streaming events are emitted in correct order (response.created, response.in_progress, response.output_item.added before MCP events) - 2. All response lifecycle events share the same response ID within a cycle - """ - if ("gpt" in model.lower() or "openai" in model.lower()) and not os.getenv( - "OPENAI_API_KEY" - ): - pytest.skip("OPENAI_API_KEY not set, skipping openai model test") - - from unittest.mock import AsyncMock, patch - - mock_mcp_tools = [ - MCPTool.model_validate({ - "name": "get_weather", - "description": "Get weather for a city", - "inputSchema": { - "type": "object", - "properties": { - "city": {"type": "string", "description": "City name"} - }, - "required": ["city"], - }, - }, by_name=False) - ] - - with caplog.at_level(logging.ERROR): - with ( - patch.object( - LiteLLM_Proxy_MCP_Handler, - "_get_mcp_tools_from_manager", - new_callable=AsyncMock, - ) as mock_get_tools, - patch.object( - LiteLLM_Proxy_MCP_Handler, - "_execute_tool_calls", - new_callable=AsyncMock, - ) as mock_execute_tools, - ): - mock_get_tools.return_value = (mock_mcp_tools, ["litellm_proxy"]) - - def mock_execute_side_effect(tool_calls, user_api_key_auth, **kwargs): - results = [] - for tool_call in tool_calls: - call_id = None - if isinstance(tool_call, dict): - call_id = tool_call.get("call_id") or tool_call.get("id") - elif hasattr(tool_call, "call_id"): - call_id = tool_call.call_id - elif hasattr(tool_call, "id"): - call_id = tool_call.id - if call_id: - results.append( - { - "tool_call_id": call_id, - "result": "Sunny, 72ยฐF", - } - ) - return results - - mock_execute_tools.side_effect = mock_execute_side_effect - - mcp_tool_config = cast( - Any, - { - "type": "mcp", - "server_url": "litellm_proxy", - "require_approval": "never", - }, - ) - - response = await litellm.aresponses( - model=model, - tools=[mcp_tool_config], - input=[ - { - "role": "user", - "type": "message", - "content": "What's the weather in San Francisco?", - } - ], - stream=True, - ) - - events = [] - async for chunk in response: - events.append(chunk) - - assert len(events) > 0, "Should receive streaming events" - - created_idx = next( - ( - i - for i, e in enumerate(events) - if getattr(e, "type", None) == "response.created" - ), - None, - ) - in_progress_idx = next( - ( - i - for i, e in enumerate(events) - if getattr(e, "type", None) == "response.in_progress" - ), - None, - ) - output_item_added_idx = next( - ( - i - for i, e in enumerate(events) - if getattr(e, "type", None) == "response.output_item.added" - ), - None, - ) - mcp_in_progress_idx = next( - ( - i - for i, e in enumerate(events) - if "mcp_list_tools.in_progress" in str(getattr(e, "type", "")) - ), - None, - ) - completed_idx = next( - ( - i - for i, e in enumerate(events) - if getattr(e, "type", None) == "response.completed" - ), - None, - ) - - assert created_idx is not None, "response.created event should be present" - assert ( - in_progress_idx is not None - ), "response.in_progress event should be present" - assert ( - output_item_added_idx is not None - ), "response.output_item.added event should be present" - - assert ( - created_idx < in_progress_idx - ), "response.created should come before response.in_progress" - assert ( - in_progress_idx < output_item_added_idx - ), "response.in_progress should come before response.output_item.added" - - if mcp_in_progress_idx is not None: - assert ( - output_item_added_idx < mcp_in_progress_idx - ), "response.output_item.added should come before response.mcp_list_tools.in_progress" - - response_ids = [] - for i, event in enumerate(events): - event_type = getattr(event, "type", None) - if hasattr(event, "response"): - response_obj = getattr(event, "response", None) - if response_obj and hasattr(response_obj, "id"): - event_type_value = ( - event_type.value - if hasattr(event_type, "value") - else str(event_type) - ) - if any( - x in event_type_value - for x in [ - "response.created", - "response.in_progress", - "response.completed", - ] - ): - response_ids.append((i, event_type_value, response_obj.id)) - - assert ( - len(response_ids) >= 2 - ), f"Should have at least 2 response lifecycle events. Found {len(response_ids)}" - - cycles = [] - current_cycle = [] - current_id = None - - for idx, event_type, resp_id in response_ids: - if current_id is None or resp_id == current_id: - current_cycle.append((idx, event_type, resp_id)) - current_id = resp_id - else: - if current_cycle: - cycles.append(current_cycle) - current_cycle = [(idx, event_type, resp_id)] - current_id = resp_id - if current_cycle: - cycles.append(current_cycle) - - for cycle_num, cycle in enumerate(cycles): - cycle_ids = set(resp_id for _, _, resp_id in cycle) - assert ( - len(cycle_ids) == 1 - ), f"Cycle {cycle_num + 1} should have consistent response ID. Found {len(cycle_ids)} unique IDs" - - assert ( - completed_idx is not None - ), "response.completed event should be present" - - lite_errors = [ - record - for record in caplog.records - if record.levelno >= logging.ERROR - and ("LiteLLM" in record.name or "LiteLLM" in record.getMessage()) - ] - assert not lite_errors, "Unexpected LiteLLM errors: " + ", ".join( - record.getMessage() for record in lite_errors - ) diff --git a/tests/proxy_unit_tests/test_skills_db.py b/tests/unit/skills/test_skills_db.py similarity index 98% rename from tests/proxy_unit_tests/test_skills_db.py rename to tests/unit/skills/test_skills_db.py index 8eb07a5ad48..20ffed6fec1 100644 --- a/tests/proxy_unit_tests/test_skills_db.py +++ b/tests/unit/skills/test_skills_db.py @@ -42,7 +42,7 @@ def create_skill_zip(skill_name: str): The zip file is automatically cleaned up after use. """ - test_dir = Path(__file__).parent.parent / "llm_translation" / "test_skills_data" + test_dir = Path(__file__).parents[2] / "llm_translation" / "test_skills_data" skill_dir = test_dir / skill_name # Create a zip file containing the skill directory diff --git a/tests/unit/skills/test_skills_main.py b/tests/unit/skills/test_skills_main.py index e1c66c8d9ea..71d65d45a08 100644 --- a/tests/unit/skills/test_skills_main.py +++ b/tests/unit/skills/test_skills_main.py @@ -30,7 +30,7 @@ def test_create_skill_forwards_description_and_instructions_from_top_level_kwarg def test_create_skill_forwards_description_and_instructions_from_extra_body(monkeypatch) -> None: - """The SDK convention (see tests/proxy_unit_tests/test_skills_db.py) nests them under + """The SDK convention (see tests/unit/skills/test_skills_db.py) nests them under extra_body instead of passing them as top-level kwargs; both paths must reach the DB.""" handler = MagicMock() monkeypatch.setattr(skills_main, "_get_litellm_skills_handler", lambda: handler)