mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
ci: move proxy, enterprise, mcp, caching, extras and gateway tests into tiers and run them from tests.yml
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
e8ce695225
commit
d3f455d65c
169 changed files with 1070 additions and 547 deletions
|
|
@ -31,7 +31,7 @@ while IFS= read -r file || [ -n "$file" ]; do
|
|||
case "$file" in
|
||||
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json)
|
||||
has_cost_map=true ;;
|
||||
tests/test_litellm/* | tests/proxy_unit_tests/*) : ;;
|
||||
tests/test_litellm/* | tests/proxy_unit_tests/* | tests/unit/proxy/*) : ;;
|
||||
*) outside_cost_map_set=true ;;
|
||||
esac
|
||||
done
|
||||
|
|
|
|||
141
.circleci/scripts/unit_selection.sh
Executable file
141
.circleci/scripts/unit_selection.sh
Executable file
|
|
@ -0,0 +1,141 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
flag="${1:?usage: unit_selection.sh <codecov flag>}"
|
||||
|
||||
legacy_flags=(
|
||||
caching-local
|
||||
enterprise-package
|
||||
enterprise-routing
|
||||
mcp-integration
|
||||
proxy-db-auth-checks
|
||||
proxy-db-budgets
|
||||
proxy-db-custom-logging
|
||||
proxy-db-db-and-spend
|
||||
proxy-db-endpoints-and-responses
|
||||
proxy-db-guardrails-hooks
|
||||
proxy-db-jwt-and-keys
|
||||
proxy-db-key-generation
|
||||
proxy-db-logging-misc
|
||||
proxy-db-proxy-runtime
|
||||
proxy-db-proxy-server-core
|
||||
proxy-db-proxy-utils
|
||||
proxy-extras
|
||||
proxy-infra
|
||||
)
|
||||
|
||||
legacy_paths() {
|
||||
case "$1" in
|
||||
caching-local) echo tests/unit/caching ;;
|
||||
enterprise-package)
|
||||
echo tests/unit/enterprise/integrations
|
||||
echo tests/unit/enterprise/proxy/auth
|
||||
echo tests/unit/enterprise/proxy/guardrails
|
||||
echo tests/unit/enterprise/proxy/hooks
|
||||
echo tests/unit/enterprise/proxy/management_endpoints
|
||||
echo tests/unit/enterprise/proxy/test_audit_logging_endpoints.py
|
||||
echo tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py ;;
|
||||
enterprise-routing)
|
||||
echo tests/unit/enterprise/enterprise_callbacks/send_emails
|
||||
echo tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py
|
||||
echo tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py
|
||||
echo tests/unit/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py
|
||||
echo tests/unit/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py
|
||||
echo tests/unit/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
|
||||
echo tests/unit/enterprise/proxy/test_deleted_file_returns_403_not_404.py
|
||||
echo tests/unit/enterprise/proxy/test_enterprise_routes.py
|
||||
echo tests/unit/enterprise/proxy/test_file_deletion_blocking.py
|
||||
echo tests/unit/enterprise/proxy/test_managed_files_access_check.py
|
||||
echo tests/unit/enterprise/proxy/test_managed_files_hook.py ;;
|
||||
mcp-integration)
|
||||
echo tests/unit/proxy/_experimental/mcp_server
|
||||
echo tests/unit/responses/mcp
|
||||
echo tests/integration/mcp_sdk1 ;;
|
||||
proxy-db-auth-checks)
|
||||
echo tests/unit/proxy/auth/test_auth_checks.py
|
||||
echo tests/unit/proxy/auth/test_user_api_key_auth.py
|
||||
echo tests/unit/proxy/test_deprecated_key_grace_period.py ;;
|
||||
proxy-db-budgets)
|
||||
echo tests/unit/proxy/auth/test_default_end_user_budget_simple.py
|
||||
echo tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
|
||||
echo tests/unit/proxy/test_zero_cost_model_budget_bypass.py ;;
|
||||
proxy-db-custom-logging)
|
||||
echo tests/unit/proxy/test_custom_callback_input.py
|
||||
echo tests/unit/proxy/test_custom_logger_s3_gcs.py ;;
|
||||
proxy-db-db-and-spend)
|
||||
echo tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
|
||||
echo tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
|
||||
echo tests/unit/proxy/db/test_update_daily_tag_spend.py
|
||||
echo tests/unit/proxy/test_db_schema_changes.py
|
||||
echo tests/unit/proxy/test_prisma_client_backoff_retry.py
|
||||
echo tests/unit/proxy/test_update_spend.py
|
||||
echo tests/unit/skills/test_skills_db.py ;;
|
||||
proxy-db-endpoints-and-responses)
|
||||
echo tests/unit/proxy/auth/test_models_fallback_endpoint.py
|
||||
echo tests/unit/proxy/common_utils/test_check_batch_cost.py
|
||||
echo tests/unit/proxy/common_utils/test_check_responses_cost.py
|
||||
echo tests/unit/proxy/common_utils/test_realtime_cache.py
|
||||
echo tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
|
||||
echo tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
|
||||
echo tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
|
||||
echo tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
|
||||
echo tests/unit/proxy/response_polling/test_response_polling_handler.py
|
||||
echo tests/unit/proxy/test_custom_tokenizer_bug.py
|
||||
echo tests/unit/proxy/test_get_favicon.py
|
||||
echo tests/unit/proxy/test_get_image.py
|
||||
echo tests/unit/proxy/test_prompt_test_endpoint.py
|
||||
echo tests/unit/proxy/test_reducto_ocr_route.py
|
||||
echo tests/unit/proxy/test_response_polling_pre_call_checks.py
|
||||
echo tests/unit/proxy/test_ui_path_detection.py ;;
|
||||
proxy-db-guardrails-hooks)
|
||||
echo tests/unit/proxy/hooks/test_banned_keyword_list.py
|
||||
echo tests/unit/proxy/test_proxy_setting_guardrails.py
|
||||
echo tests/unit/proxy/test_unit_test_proxy_hooks.py ;;
|
||||
proxy-db-jwt-and-keys)
|
||||
echo tests/unit/proxy/auth/test_jwt.py
|
||||
echo tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
|
||||
echo tests/unit/proxy/test_proxy_custom_auth.py ;;
|
||||
proxy-db-key-generation) echo tests/unit/proxy/management_endpoints/test_key_generate_prisma.py ;;
|
||||
proxy-db-logging-misc)
|
||||
echo tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
|
||||
echo tests/unit/proxy/spend_tracking/test_search_api_logging.py
|
||||
echo tests/unit/proxy/test_proxy_reject_logging.py ;;
|
||||
proxy-db-proxy-runtime)
|
||||
echo tests/unit/proxy/auth/test_multipart_bypass_repro.py
|
||||
echo tests/unit/proxy/auth/test_proxy_routes.py
|
||||
echo tests/unit/proxy/middleware/test_request_size_limit_middleware.py
|
||||
echo tests/unit/proxy/test_proxy_config_unit_test.py
|
||||
echo tests/unit/proxy/test_proxy_pass_user_config.py
|
||||
echo tests/unit/proxy/test_proxy_token_counter.py
|
||||
echo tests/unit/proxy/test_server_root_path.py ;;
|
||||
proxy-db-proxy-server-core)
|
||||
echo tests/unit/proxy/test_aproxy_startup.py
|
||||
echo tests/unit/proxy/test_proxy_server.py ;;
|
||||
proxy-db-proxy-utils) echo tests/unit/proxy/test_proxy_utils.py ;;
|
||||
proxy-extras) echo tests/unit/litellm_proxy_extras ;;
|
||||
proxy-infra) echo tests/unit/gateway ;;
|
||||
*) echo "unit_selection.sh: unknown flag $1" >&2; exit 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
expand() {
|
||||
while read -r path; do
|
||||
if [ -d "$path" ]; then
|
||||
find "$path" -name 'test_*.py'
|
||||
elif [ -f "$path" ]; then
|
||||
echo "$path"
|
||||
else
|
||||
echo "unit_selection.sh: $path does not exist" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
if [ "$flag" = unit ]; then
|
||||
comm -23 \
|
||||
<(find tests/unit -name 'test_*.py' | sort) \
|
||||
<(for legacy in "${legacy_flags[@]}"; do legacy_paths "$legacy"; done | expand | sort)
|
||||
exit 0
|
||||
fi
|
||||
|
||||
legacy_paths "$flag" | expand | sort
|
||||
|
|
@ -165,21 +165,27 @@ commands:
|
|||
jobs:
|
||||
unit:
|
||||
parameters:
|
||||
tests_path:
|
||||
type: string
|
||||
default: tests/unit
|
||||
flag:
|
||||
type: string
|
||||
default: unit
|
||||
shards:
|
||||
type: integer
|
||||
default: 6
|
||||
workers:
|
||||
type: integer
|
||||
default: 4
|
||||
dist:
|
||||
type: string
|
||||
default: loadscope
|
||||
base_ref:
|
||||
type: string
|
||||
default: ""
|
||||
pull_request_url:
|
||||
type: string
|
||||
default: ""
|
||||
legacy_mcp_peer:
|
||||
type: boolean
|
||||
default: false
|
||||
machine:
|
||||
image: ubuntu-2204:2024.04.1
|
||||
resource_class: large
|
||||
|
|
@ -194,15 +200,26 @@ jobs:
|
|||
base_ref: << parameters.base_ref >>
|
||||
pull_request_url: << parameters.pull_request_url >>
|
||||
- setup_test_deps
|
||||
- when:
|
||||
condition: << parameters.legacy_mcp_peer >>
|
||||
steps:
|
||||
- run:
|
||||
name: Install MCP SDK1 peer
|
||||
command: |
|
||||
uv venv --python 3.12 .venv-mcp-peer
|
||||
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
||||
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
|
||||
- run:
|
||||
name: "Run << parameters.tests_path >> shard"
|
||||
name: "Run << parameters.flag >> shard"
|
||||
no_output_timeout: 20m
|
||||
command: |
|
||||
mkdir -p test-results/<< parameters.flag >>
|
||||
mapfile -t files < <(find << parameters.tests_path >> -name 'test_*.py' | sort | circleci tests split --split-by=timings --timings-type=filename)
|
||||
if [ "${#files[@]}" -eq 0 ]; then echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.tests_path >> files; nothing to run"; exit 0; fi
|
||||
mapfile -t files < <(bash .circleci/scripts/unit_selection.sh << parameters.flag >> | circleci tests split --split-by=timings --timings-type=filename)
|
||||
if [ "${#files[@]}" -eq 0 ]; then echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.flag >> files; nothing to run"; exit 0; fi
|
||||
xdist_args=()
|
||||
if [ "<< parameters.workers >>" -gt 0 ]; then xdist_args=(-n << parameters.workers >> --dist=<< parameters.dist >>); fi
|
||||
set +e
|
||||
uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml
|
||||
uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 "${xdist_args[@]}" --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml
|
||||
status=$?
|
||||
set -e
|
||||
if [ "$status" -eq 5 ]; then echo "pytest collected no tests from the shard; passing"; exit 0; fi
|
||||
|
|
@ -286,6 +303,57 @@ workflows:
|
|||
- unit:
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- unit:
|
||||
name: unit-<< matrix.flag >>
|
||||
shards: 1
|
||||
workers: 2
|
||||
matrix:
|
||||
parameters:
|
||||
flag: [caching-local, proxy-extras, enterprise-routing]
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- unit:
|
||||
name: unit-mcp-integration
|
||||
flag: mcp-integration
|
||||
shards: 1
|
||||
workers: 2
|
||||
legacy_mcp_peer: true
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- unit:
|
||||
name: unit-<< matrix.flag >>
|
||||
shards: 1
|
||||
matrix:
|
||||
parameters:
|
||||
flag:
|
||||
- enterprise-package
|
||||
- proxy-infra
|
||||
- proxy-db-auth-checks
|
||||
- proxy-db-jwt-and-keys
|
||||
- proxy-db-proxy-server-core
|
||||
- proxy-db-proxy-runtime
|
||||
- proxy-db-custom-logging
|
||||
- proxy-db-logging-misc
|
||||
- proxy-db-db-and-spend
|
||||
- proxy-db-guardrails-hooks
|
||||
- proxy-db-budgets
|
||||
- proxy-db-endpoints-and-responses
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- unit:
|
||||
name: unit-proxy-db-proxy-utils
|
||||
flag: proxy-db-proxy-utils
|
||||
shards: 1
|
||||
dist: worksteal
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- unit:
|
||||
name: unit-proxy-db-key-generation
|
||||
flag: proxy-db-key-generation
|
||||
shards: 1
|
||||
workers: 0
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- documentation
|
||||
- integration:
|
||||
name: integration-<< matrix.suite >>
|
||||
|
|
|
|||
13
.github/scripts/assert_ci_coverage.py
vendored
13
.github/scripts/assert_ci_coverage.py
vendored
|
|
@ -34,7 +34,6 @@ GLOB_CHARS = frozenset("*?")
|
|||
# tests has to be named by some shard or it runs nowhere. A child listed here is
|
||||
# itself decomposed one level deeper and is checked through its own entry.
|
||||
SHARDED_ROOTS: tuple[str, ...] = (
|
||||
"tests/proxy_unit_tests",
|
||||
"tests/test_litellm",
|
||||
"tests/test_litellm/proxy",
|
||||
)
|
||||
|
|
@ -120,6 +119,13 @@ def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
|
|||
)
|
||||
|
||||
|
||||
def _unit_selection_tokens(repo_root: pathlib.Path = REPO_ROOT) -> frozenset[str]:
|
||||
script: Final = repo_root / ".circleci/scripts/unit_selection.sh"
|
||||
if not script.is_file():
|
||||
return frozenset()
|
||||
return frozenset(match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(_uncommented(script.read_text())))
|
||||
|
||||
|
||||
def _built_dockerfile_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
|
||||
return frozenset(
|
||||
match.group(0)
|
||||
|
|
@ -611,7 +617,10 @@ def main() -> int:
|
|||
scalars = _all_scalars()
|
||||
|
||||
integration_paths, ownership_findings = _integration_ownership()
|
||||
test_findings = _uncovered_tests(allowlist, _invoked_test_tokens(scalars) | integration_paths) + ownership_findings
|
||||
test_findings = (
|
||||
_uncovered_tests(allowlist, _invoked_test_tokens(scalars) | _unit_selection_tokens() | integration_paths)
|
||||
+ ownership_findings
|
||||
)
|
||||
dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
|
||||
stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
|
||||
|
||||
|
|
|
|||
33
.github/workflows/_test-unit-base.yml
vendored
33
.github/workflows/_test-unit-base.yml
vendored
|
|
@ -13,6 +13,15 @@ on:
|
|||
have its path existence-checked like any other token.
|
||||
required: true
|
||||
type: string
|
||||
fork-flag:
|
||||
description: >-
|
||||
Codecov flag of the `.circleci/tests.yml` job that now owns part of
|
||||
this shard. CircleCI does not run on pull requests from forks, so on
|
||||
those events this shard also runs the files
|
||||
`.circleci/scripts/unit_selection.sh` lists for the flag.
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
workers:
|
||||
description: "Number of pytest-xdist workers"
|
||||
required: false
|
||||
|
|
@ -92,6 +101,7 @@ jobs:
|
|||
pull-requests: read
|
||||
outputs:
|
||||
decision: ${{ steps.changes.outputs.decision }}
|
||||
has-coverage: ${{ steps.tests.outputs.has-coverage }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
|
|
@ -160,10 +170,13 @@ jobs:
|
|||
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
||||
|
||||
- name: Run tests
|
||||
id: tests
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: ${{ inputs.timeout-minutes }}
|
||||
env:
|
||||
TEST_PATH: ${{ inputs.test-path }}
|
||||
FORK_FLAG: ${{ inputs.fork-flag }}
|
||||
IS_FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}
|
||||
MAX_FAILURES: ${{ inputs.max-failures }}
|
||||
WORKERS: ${{ inputs.workers }}
|
||||
RERUNS: ${{ inputs.reruns }}
|
||||
|
|
@ -171,9 +184,18 @@ jobs:
|
|||
DIST: ${{ inputs.dist }}
|
||||
COVERAGE_CORE: sysmon
|
||||
run: |
|
||||
echo "has-coverage=false" >> "$GITHUB_OUTPUT"
|
||||
selection="${TEST_PATH}"
|
||||
if [ "${IS_FORK}" = "true" ] && [ -n "${FORK_FLAG}" ]; then
|
||||
selection="${TEST_PATH} $(bash .circleci/scripts/unit_selection.sh "${FORK_FLAG}" | tr '\n' ' ')"
|
||||
fi
|
||||
if [ -z "${selection// /}" ]; then
|
||||
echo "shard selection is empty on this event (CircleCI flag ${FORK_FLAG:-none} owns it); nothing to run"
|
||||
exit 0
|
||||
fi
|
||||
pytest_args=()
|
||||
existing_paths=0
|
||||
for token in ${TEST_PATH:?}; do
|
||||
for token in ${selection}; do
|
||||
case "${token}" in
|
||||
-*) pytest_args+=("${token}") ;;
|
||||
*)
|
||||
|
|
@ -187,7 +209,7 @@ jobs:
|
|||
esac
|
||||
done
|
||||
if [ "${existing_paths}" -eq 0 ]; then
|
||||
echo "No path in TEST_PATH exists (${TEST_PATH}); nothing to run"
|
||||
echo "No path in the selection exists (${selection}); nothing to run"
|
||||
exit 0
|
||||
fi
|
||||
xdist_args=()
|
||||
|
|
@ -209,8 +231,11 @@ jobs:
|
|||
--cov-config=pyproject.toml
|
||||
status=$?
|
||||
set -e
|
||||
if [ -f coverage.xml ]; then
|
||||
echo "has-coverage=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
if [ "$status" -eq 5 ]; then
|
||||
echo "pytest collected no tests from ${TEST_PATH}; passing"
|
||||
echo "pytest collected no tests from ${selection}; passing"
|
||||
exit 0
|
||||
fi
|
||||
exit "$status"
|
||||
|
|
@ -226,7 +251,7 @@ jobs:
|
|||
upload-coverage:
|
||||
name: Upload coverage to Codecov
|
||||
needs: run
|
||||
if: always() && needs.run.outputs.decision != 'skip'
|
||||
if: always() && needs.run.outputs.decision != 'skip' && needs.run.outputs.has-coverage == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
97
.github/workflows/test-unit-proxy-db.yml
vendored
97
.github/workflows/test-unit-proxy-db.yml
vendored
|
|
@ -20,6 +20,12 @@ concurrency:
|
|||
# rather than alphabetical letter ranges. Adding a new test file means adding it
|
||||
# to whichever group it belongs to, not reshuffling slices.
|
||||
#
|
||||
# `.circleci/tests.yml` runs each group's files on same-repo events under the
|
||||
# `proxy-db-<group>` Codecov flag; `.circleci/scripts/unit_selection.sh` holds
|
||||
# the file lists. CircleCI does not build pull requests from forks, so `fork-flag`
|
||||
# makes the shard run that list there. `test-path` keeps the files that still
|
||||
# reach real providers and never left tests/proxy_unit_tests.
|
||||
#
|
||||
# Design targets:
|
||||
# * Every shard runs in <= 7 minutes of wall-clock on the default runner.
|
||||
# Most of a shard's time is pytest plugin load + xdist worker imports +
|
||||
|
|
@ -58,7 +64,7 @@ jobs:
|
|||
proxy-db:
|
||||
needs: assert-shard-coverage
|
||||
# Display only the semantic shard name in the checks UI instead of GHA's
|
||||
# default "proxy-db (key-generation, tests/proxy_unit_tests/…, 0, loadscope, 20)"
|
||||
# default "proxy-db (key-generation, tests/unit/proxy/…, 0, loadscope, 20)"
|
||||
# which includes every matrix field and gets truncated past the test-path.
|
||||
name: ${{ matrix.test-group }}
|
||||
permissions:
|
||||
|
|
@ -71,132 +77,93 @@ jobs:
|
|||
include:
|
||||
# Must run serially — event-loop conflict with the logging worker.
|
||||
- test-group: key-generation
|
||||
test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py"
|
||||
test-path: ""
|
||||
fork-flag: proxy-db-key-generation
|
||||
workers: 0
|
||||
dist: loadscope
|
||||
timeout: 20
|
||||
|
||||
# ---- auth: split into 2 shards ----
|
||||
- test-group: auth-checks
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_auth_checks.py
|
||||
tests/proxy_unit_tests/test_user_api_key_auth.py
|
||||
tests/proxy_unit_tests/test_deprecated_key_grace_period.py
|
||||
test-path: ""
|
||||
fork-flag: proxy-db-auth-checks
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
- test-group: jwt-and-keys
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_jwt.py
|
||||
tests/proxy_unit_tests/test_jwt_key_mapping.py
|
||||
tests/proxy_unit_tests/test_proxy_custom_auth.py
|
||||
tests/proxy_unit_tests/test_key_generate_dynamodb.py
|
||||
test-path: ""
|
||||
fork-flag: proxy-db-jwt-and-keys
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
|
||||
# ---- test_proxy_utils.py, single shard, worksteal distribution ----
|
||||
- test-group: proxy-utils
|
||||
test-path: "tests/proxy_unit_tests/test_proxy_utils.py"
|
||||
test-path: ""
|
||||
fork-flag: proxy-db-proxy-utils
|
||||
workers: 4
|
||||
dist: worksteal
|
||||
timeout: 15
|
||||
|
||||
# ---- proxy server: split into 2 shards ----
|
||||
- test-group: proxy-server-core
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_proxy_server.py
|
||||
tests/proxy_unit_tests/test_aproxy_startup.py
|
||||
test-path: "tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py"
|
||||
fork-flag: proxy-db-proxy-server-core
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
- test-group: proxy-runtime
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_proxy_config_unit_test.py
|
||||
tests/proxy_unit_tests/test_proxy_routes.py
|
||||
tests/proxy_unit_tests/test_server_root_path.py
|
||||
tests/proxy_unit_tests/test_proxy_pass_user_config.py
|
||||
tests/proxy_unit_tests/test_proxy_token_counter.py
|
||||
tests/proxy_unit_tests/test_request_size_limit_middleware.py
|
||||
tests/proxy_unit_tests/test_multipart_bypass_repro.py
|
||||
test-path: ""
|
||||
fork-flag: proxy-db-proxy-runtime
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
|
||||
# ---- logging: split into 2 shards ----
|
||||
- test-group: custom-logging
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_custom_callback_input.py
|
||||
tests/proxy_unit_tests/test_custom_logger_s3_gcs.py
|
||||
tests/proxy_unit_tests/test_proxy_custom_logger.py
|
||||
test-path: "tests/proxy_unit_tests/test_proxy_custom_logger.py"
|
||||
fork-flag: proxy-db-custom-logging
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
- test-group: logging-misc
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_proxy_reject_logging.py
|
||||
tests/proxy_unit_tests/test_audit_logs_proxy.py
|
||||
tests/proxy_unit_tests/test_search_api_logging.py
|
||||
test-path: ""
|
||||
fork-flag: proxy-db-logging-misc
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
|
||||
- test-group: db-and-spend
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_prisma_client_backoff_retry.py
|
||||
tests/proxy_unit_tests/test_db_schema_changes.py
|
||||
tests/proxy_unit_tests/test_e2e_pod_lock_manager.py
|
||||
tests/proxy_unit_tests/test_skills_db.py
|
||||
tests/proxy_unit_tests/test_update_daily_tag_spend.py
|
||||
tests/proxy_unit_tests/test_update_spend.py
|
||||
tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py
|
||||
test-path: ""
|
||||
fork-flag: proxy-db-db-and-spend
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
|
||||
# ---- guardrails + budget + hooks: split into 2 ----
|
||||
- test-group: guardrails-hooks
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_proxy_setting_guardrails.py
|
||||
tests/proxy_unit_tests/test_banned_keyword_list.py
|
||||
tests/proxy_unit_tests/test_unit_test_proxy_hooks.py
|
||||
test-path: ""
|
||||
fork-flag: proxy-db-guardrails-hooks
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
- test-group: budgets
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_default_end_user_budget_simple.py
|
||||
tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py
|
||||
tests/proxy_unit_tests/test_zero_cost_model_budget_bypass.py
|
||||
test-path: ""
|
||||
fork-flag: proxy-db-budgets
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
|
||||
- test-group: endpoints-and-responses
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_blog_posts_endpoint.py
|
||||
tests/proxy_unit_tests/test_models_fallback_endpoint.py
|
||||
tests/proxy_unit_tests/test_google_endpoint_routing.py
|
||||
tests/proxy_unit_tests/test_google_gemini_proxy_request.py
|
||||
tests/proxy_unit_tests/test_gemini_agents_endpoints.py
|
||||
tests/proxy_unit_tests/test_get_favicon.py
|
||||
tests/proxy_unit_tests/test_get_image.py
|
||||
tests/proxy_unit_tests/test_reducto_ocr_route.py
|
||||
tests/proxy_unit_tests/test_ui_path_detection.py
|
||||
tests/proxy_unit_tests/test_prompt_test_endpoint.py
|
||||
tests/proxy_unit_tests/test_check_batch_cost.py
|
||||
tests/proxy_unit_tests/test_check_responses_cost.py
|
||||
tests/proxy_unit_tests/test_response_polling_handler.py
|
||||
tests/proxy_unit_tests/test_response_polling_pre_call_checks.py
|
||||
tests/proxy_unit_tests/test_realtime_cache.py
|
||||
tests/proxy_unit_tests/test_proxy_exception_mapping.py
|
||||
tests/proxy_unit_tests/test_custom_tokenizer_bug.py
|
||||
test-path: "tests/proxy_unit_tests/test_proxy_exception_mapping.py"
|
||||
fork-flag: proxy-db-endpoints-and-responses
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
uses: ./.github/workflows/_test-unit-base.yml
|
||||
with:
|
||||
test-path: ${{ matrix.test-path }}
|
||||
fork-flag: ${{ matrix.fork-flag }}
|
||||
workers: ${{ matrix.workers }}
|
||||
reruns: 2
|
||||
timeout-minutes: ${{ matrix.timeout }}
|
||||
|
|
|
|||
31
.github/workflows/test-unit.yml
vendored
31
.github/workflows/test-unit.yml
vendored
|
|
@ -31,10 +31,14 @@ concurrency:
|
|||
# number, so a partially-specified entry would fail the call rather than fall
|
||||
# back to the default.
|
||||
#
|
||||
# tests/proxy_unit_tests keeps its own caller (test-unit-proxy-db.yml): it is
|
||||
# already a matrix and carries a shard-coverage guard that reads that file by
|
||||
# name. Folding it in here is a follow-up, together with generalising that guard
|
||||
# into assert_ci_coverage.py.
|
||||
# tests/unit/proxy keeps its own caller (test-unit-proxy-db.yml): it is already
|
||||
# a matrix and carries a shard-coverage guard that reads that file by name.
|
||||
# Folding it in here is a follow-up, together with generalising that guard into
|
||||
# assert_ci_coverage.py.
|
||||
#
|
||||
# `fork-flag` names the `.circleci/tests.yml` job that now runs part of the
|
||||
# shard under the same Codecov flag. CircleCI does not build pull requests from
|
||||
# forks, so the shard still runs those files there and skips them elsewhere.
|
||||
jobs:
|
||||
unit:
|
||||
name: ${{ matrix.shard }}
|
||||
|
|
@ -49,6 +53,7 @@ jobs:
|
|||
- shard: mcp-integration
|
||||
artifact-name: mcp-integration
|
||||
test-path: "tests/mcp_tests tests/test_litellm/experimental_mcp_client"
|
||||
fork-flag: mcp-integration
|
||||
workers: 2
|
||||
reruns: 0
|
||||
timeout-minutes: 20
|
||||
|
|
@ -65,10 +70,10 @@ jobs:
|
|||
- shard: enterprise-routing
|
||||
artifact-name: enterprise-routing
|
||||
test-path: >-
|
||||
tests/test_litellm/enterprise
|
||||
tests/test_litellm/google_genai
|
||||
tests/test_litellm/router_utils
|
||||
tests/test_litellm/router_strategy
|
||||
fork-flag: enterprise-routing
|
||||
workers: 2
|
||||
reruns: 2
|
||||
timeout-minutes: 20
|
||||
|
|
@ -199,7 +204,7 @@ jobs:
|
|||
tests/test_litellm/proxy/types_utils
|
||||
tests/test_litellm/proxy/logging_endpoints
|
||||
tests/test_litellm/proxy/test_*.py
|
||||
tests/test_gateway
|
||||
fork-flag: proxy-infra
|
||||
workers: 4
|
||||
reruns: 2
|
||||
timeout-minutes: 20
|
||||
|
|
@ -207,11 +212,8 @@ jobs:
|
|||
|
||||
- shard: caching-local
|
||||
artifact-name: caching-local
|
||||
test-path: >-
|
||||
tests/local_testing/test_cache_preset_key.py
|
||||
tests/local_testing/test_caching_handler.py
|
||||
tests/local_testing/test_responses_stream_cache_keys.py
|
||||
tests/local_testing/test_unit_test_caching.py
|
||||
test-path: ""
|
||||
fork-flag: caching-local
|
||||
workers: 2
|
||||
reruns: 2
|
||||
timeout-minutes: 20
|
||||
|
|
@ -219,7 +221,8 @@ jobs:
|
|||
|
||||
- shard: proxy-extras
|
||||
artifact-name: proxy-extras
|
||||
test-path: "tests/litellm-proxy-extras"
|
||||
test-path: ""
|
||||
fork-flag: proxy-extras
|
||||
workers: 2
|
||||
reruns: 2
|
||||
timeout-minutes: 20
|
||||
|
|
@ -227,7 +230,8 @@ jobs:
|
|||
|
||||
- shard: enterprise-package
|
||||
artifact-name: enterprise-package
|
||||
test-path: "tests/enterprise"
|
||||
test-path: ""
|
||||
fork-flag: enterprise-package
|
||||
workers: 4
|
||||
reruns: 2
|
||||
timeout-minutes: 20
|
||||
|
|
@ -246,6 +250,7 @@ jobs:
|
|||
uses: ./.github/workflows/_test-unit-base.yml
|
||||
with:
|
||||
test-path: ${{ matrix.test-path }}
|
||||
fork-flag: ${{ matrix.fork-flag || '' }}
|
||||
workers: ${{ matrix.workers }}
|
||||
reruns: ${{ matrix.reruns }}
|
||||
timeout-minutes: ${{ matrix.timeout-minutes }}
|
||||
|
|
|
|||
12
Makefile
12
Makefile
|
|
@ -51,8 +51,8 @@ help:
|
|||
@echo " make test-unit-core-utils - Run core utils tests (~32 files)"
|
||||
@echo " make test-unit-other - Run other tests (caching, responses, etc., ~69 files)"
|
||||
@echo " make test-unit-root - Run root-level tests (~34 files)"
|
||||
@echo " make test-proxy-unit-a - Run proxy_unit_tests (a-o, ~20 files)"
|
||||
@echo " make test-proxy-unit-b - Run proxy_unit_tests (p-z, ~28 files)"
|
||||
@echo " make test-proxy-unit-a - Run tests/unit/proxy (a-o)"
|
||||
@echo " make test-proxy-unit-b - Run tests/unit/proxy (p-z)"
|
||||
@echo " make test-integration - Run integration tests"
|
||||
@echo " make test-unit-helm - Run helm unit tests"
|
||||
@echo " make test-rust-extension - Build the Rust extension and run its public Python tests"
|
||||
|
|
@ -332,17 +332,17 @@ test-unit-core-utils: install-test-deps
|
|||
$(UV_RUN) pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20
|
||||
|
||||
test-unit-other: install-test-deps
|
||||
$(UV_RUN) pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20
|
||||
$(UV_RUN) pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/unit/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20
|
||||
|
||||
test-unit-root: install-test-deps
|
||||
$(UV_RUN) pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20
|
||||
|
||||
# Proxy unit tests (tests/proxy_unit_tests split alphabetically)
|
||||
# Proxy unit tests (tests/unit/proxy split alphabetically)
|
||||
test-proxy-unit-a: install-test-deps
|
||||
$(UV_RUN) pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20
|
||||
$(UV_RUN) pytest tests/unit/proxy/test_[a-o]*.py --tb=short -vv -n 2 --durations=20
|
||||
|
||||
test-proxy-unit-b: install-test-deps
|
||||
$(UV_RUN) pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20
|
||||
$(UV_RUN) pytest tests/unit/proxy/test_[p-z]*.py --tb=short -vv -n 2 --durations=20
|
||||
|
||||
test-integration: install-test-deps
|
||||
$(UV_RUN) pytest tests/ -k "not test_litellm"
|
||||
|
|
|
|||
|
|
@ -369,7 +369,7 @@ model LiteLLM_SkillsTable {
|
|||
Run the tests:
|
||||
|
||||
```bash
|
||||
pytest tests/proxy_unit_tests/test_skills_db.py -v
|
||||
pytest tests/unit/skills/test_skills_db.py -v
|
||||
```
|
||||
|
||||
Tests cover:
|
||||
|
|
|
|||
|
|
@ -31,11 +31,11 @@ def get_function_names_from_file(file_path):
|
|||
def get_all_functions_called_in_tests(base_dir):
|
||||
"""
|
||||
Returns a set of function names that are called in test functions
|
||||
inside 'local_testing' and 'proxy_unit_tests' directories,
|
||||
inside 'local_testing' and 'unit/proxy' directories,
|
||||
specifically in files containing the word 'router'.
|
||||
"""
|
||||
called_functions = set()
|
||||
test_dirs = ["local_testing", "proxy_unit_tests"]
|
||||
test_dirs = ["local_testing", "unit/proxy"]
|
||||
|
||||
for test_dir in test_dirs:
|
||||
dir_path = os.path.join(base_dir, test_dir)
|
||||
|
|
|
|||
53
tests/integration/mcp_sdk1/conftest.py
Normal file
53
tests/integration/mcp_sdk1/conftest.py
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import asyncio
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def event_loop():
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
yield loop
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
def setup_and_teardown():
|
||||
"""
|
||||
This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained.
|
||||
"""
|
||||
importlib.reload(litellm)
|
||||
import asyncio
|
||||
|
||||
loop = asyncio.get_event_loop_policy().new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
yield
|
||||
|
||||
# Teardown code (executes after the yield point)
|
||||
# LoggingWorker carries still-queued coroutines onto the next test's loop, where they'd log into that test's callbacks
|
||||
asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue())
|
||||
loop.close() # Close the loop created earlier
|
||||
asyncio.set_event_loop(None) # Remove the reference to the loop
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
async def drain_logging_worker():
|
||||
"""
|
||||
The logging queue is bound to the running loop, so anything left queued when a test's loop
|
||||
goes away is carried onto the next test's loop and fires against its callbacks.
|
||||
"""
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
yield
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.clear_queue(), timeout=10)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
|
|
@ -17,7 +17,7 @@ mcp_servers:
|
|||
transport: stdio
|
||||
command: python3
|
||||
args:
|
||||
- tests/mcp_tests/mcp_server.py
|
||||
- tests/integration/mcp_sdk1/mcp_math_server.py
|
||||
allow_all_keys: true # Needed for master_key auth when DB/object_permission not used
|
||||
math_streamable_http:
|
||||
transport: http
|
||||
|
|
@ -35,10 +35,10 @@ from litellm.proxy.proxy_server import (
|
|||
initialize,
|
||||
)
|
||||
|
||||
CONFIG_TEMPLATE_PATH = Path("tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml")
|
||||
MCP_SERVER_SCRIPT = Path("tests/mcp_tests/mcp_server.py")
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
CONFIG_TEMPLATE_PATH = PROJECT_ROOT / "tests/integration/mcp_sdk1/test_config_mcp_e2e.yaml"
|
||||
MCP_SERVER_SCRIPT = PROJECT_ROOT / "tests/integration/mcp_sdk1/mcp_math_server.py"
|
||||
MCP_PEER_PYTHON = os.environ.get("MCP_TEST_PEER_PYTHON", sys.executable)
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
PROXY_START_TIMEOUT = 30
|
||||
|
||||
|
||||
401
tests/mcp_tests/test_aresponses_api_with_mcp_providers.py
Normal file
401
tests/mcp_tests/test_aresponses_api_with_mcp_providers.py
Normal file
|
|
@ -0,0 +1,401 @@
|
|||
import logging
|
||||
import os
|
||||
import pytest
|
||||
from mcp.types import Tool as MCPTool
|
||||
from typing import Any, cast
|
||||
|
||||
|
||||
# Import required modules
|
||||
import litellm
|
||||
from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler
|
||||
|
||||
|
||||
class MockUserAPIKeyAuth:
|
||||
"""Mock UserAPIKeyAuth for testing"""
|
||||
|
||||
def __init__(self):
|
||||
self.api_key = "test_key"
|
||||
self.user_id = "test_user"
|
||||
self.team_id = "test_team"
|
||||
self.user_email = "test@example.com"
|
||||
self.max_budget = 100.0
|
||||
self.spend = 0.0
|
||||
self.models = []
|
||||
self.aliases = {}
|
||||
self.config = {}
|
||||
self.permissions = {}
|
||||
self.metadata = {}
|
||||
self.object_permission_id = "test_permission_id"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
pytest.param("gpt-4o-mini", id="openai"),
|
||||
pytest.param("claude-haiku-4-5", id="anthropic"),
|
||||
],
|
||||
)
|
||||
async def test_streaming_responses_api_with_mcp_tools(
|
||||
model: str, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
"""
|
||||
Test the streaming responses API with MCP tools when using server_url="litellm_proxy"
|
||||
|
||||
Under the hood the follow occurs
|
||||
|
||||
- MCP: responses called litellm MCP manager.list_tools (MOCKED)
|
||||
- Request 1: Made to model under test with fetched tools (REAL LLM CALL)
|
||||
- MCP: Execute tool call from request 1 and returns result (MOCKED)
|
||||
- Request 2: Made to model under test with fetched tools and tool results (REAL LLM CALL)
|
||||
|
||||
Return the user the result of request 2
|
||||
"""
|
||||
# Skip test if API keys are not set for the respective models
|
||||
if ("claude" in model.lower() or "anthropic" in model.lower()) and not os.getenv(
|
||||
"ANTHROPIC_API_KEY"
|
||||
):
|
||||
pytest.skip("ANTHROPIC_API_KEY not set, skipping anthropic model test")
|
||||
if ("gpt" in model.lower() or "openai" in model.lower()) and not os.getenv(
|
||||
"OPENAI_API_KEY"
|
||||
):
|
||||
pytest.skip("OPENAI_API_KEY not set, skipping openai model test")
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
print("🧪 Testing basic streaming with MCP tools...")
|
||||
|
||||
# Mock MCP tools that would be returned from the manager
|
||||
mock_mcp_tools = [
|
||||
MCPTool.model_validate({
|
||||
"name": "search_repo",
|
||||
"description": "Search BerriAI/litellm repository for information",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "Search query"}
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
}, by_name=False)
|
||||
]
|
||||
|
||||
# Only mock the MCP-specific operations, let LLM responses be real
|
||||
with caplog.at_level(logging.ERROR):
|
||||
with (
|
||||
patch.object(
|
||||
LiteLLM_Proxy_MCP_Handler,
|
||||
"_get_mcp_tools_from_manager",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_get_tools,
|
||||
patch.object(
|
||||
LiteLLM_Proxy_MCP_Handler,
|
||||
"_execute_tool_calls",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_execute_tools,
|
||||
):
|
||||
# Setup MCP mocks only
|
||||
mock_get_tools.return_value = (mock_mcp_tools, ["litellm_proxy"])
|
||||
|
||||
# Create a dynamic mock that will match the actual tool call ID from the LLM response
|
||||
def mock_execute_tool_calls_side_effect(
|
||||
tool_calls, user_api_key_auth, **kwargs
|
||||
):
|
||||
"""Mock function that returns results matching the actual tool call IDs from the LLM"""
|
||||
results = []
|
||||
for tool_call in tool_calls:
|
||||
# Extract call_id from the tool call
|
||||
call_id = None
|
||||
if isinstance(tool_call, dict):
|
||||
call_id = tool_call.get("call_id") or tool_call.get("id")
|
||||
elif hasattr(tool_call, "call_id"):
|
||||
call_id = tool_call.call_id
|
||||
elif hasattr(tool_call, "id"):
|
||||
call_id = tool_call.id
|
||||
|
||||
if call_id:
|
||||
results.append(
|
||||
{
|
||||
"tool_call_id": call_id,
|
||||
"result": "LiteLLM is a unified interface for 100+ LLMs that translates inputs to provider-specific completion endpoints and provides consistent OpenAI-format output.",
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
mock_execute_tools.side_effect = mock_execute_tool_calls_side_effect
|
||||
|
||||
# Make the actual call - LLM responses will be real
|
||||
mcp_tool_config = cast(
|
||||
Any,
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_url": "litellm_proxy",
|
||||
"require_approval": "never",
|
||||
},
|
||||
)
|
||||
response = await litellm.aresponses(
|
||||
model=model,
|
||||
tools=[mcp_tool_config],
|
||||
tool_choice="required",
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"type": "message",
|
||||
"content": "give me a TLDR of what BerriAI/litellm is about",
|
||||
}
|
||||
],
|
||||
stream=True,
|
||||
)
|
||||
|
||||
print(f"📋 Response type: {type(response)}")
|
||||
assert hasattr(
|
||||
response, "__aiter__"
|
||||
), "Response should be an async streaming response"
|
||||
|
||||
# Collect streaming chunks
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
print(f"📦 Chunk type: {getattr(chunk, 'type', 'unknown')}")
|
||||
|
||||
print(f"📊 Total chunks received: {len(chunks)}")
|
||||
|
||||
# Verify MCP mocks were called (may be called multiple times in streaming)
|
||||
assert (
|
||||
mock_get_tools.call_count >= 1
|
||||
), f"Expected MCP tools to be fetched at least once, got {mock_get_tools.call_count}"
|
||||
print(f"MCP tools fetched: {len(mock_mcp_tools)}")
|
||||
|
||||
# Verify we got a response
|
||||
assert response is not None
|
||||
assert len(chunks) > 0, "Should have received streaming chunks"
|
||||
|
||||
print("Basic streaming responses API with MCP tools test passed!")
|
||||
|
||||
lite_errors = [
|
||||
record
|
||||
for record in caplog.records
|
||||
if record.levelno >= logging.ERROR
|
||||
and ("LiteLLM" in record.name or "LiteLLM" in record.getMessage())
|
||||
]
|
||||
assert not lite_errors, "Unexpected LiteLLM errors: " + ", ".join(
|
||||
record.getMessage() for record in lite_errors
|
||||
)
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model", ["gpt-4o-mini"])
|
||||
async def test_streaming_mcp_event_order_and_response_id_consistency(
|
||||
model: str, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
"""
|
||||
Test that:
|
||||
1. Streaming events are emitted in correct order (response.created, response.in_progress, response.output_item.added before MCP events)
|
||||
2. All response lifecycle events share the same response ID within a cycle
|
||||
"""
|
||||
if ("gpt" in model.lower() or "openai" in model.lower()) and not os.getenv(
|
||||
"OPENAI_API_KEY"
|
||||
):
|
||||
pytest.skip("OPENAI_API_KEY not set, skipping openai model test")
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
mock_mcp_tools = [
|
||||
MCPTool.model_validate({
|
||||
"name": "get_weather",
|
||||
"description": "Get weather for a city",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string", "description": "City name"}
|
||||
},
|
||||
"required": ["city"],
|
||||
},
|
||||
}, by_name=False)
|
||||
]
|
||||
|
||||
with caplog.at_level(logging.ERROR):
|
||||
with (
|
||||
patch.object(
|
||||
LiteLLM_Proxy_MCP_Handler,
|
||||
"_get_mcp_tools_from_manager",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_get_tools,
|
||||
patch.object(
|
||||
LiteLLM_Proxy_MCP_Handler,
|
||||
"_execute_tool_calls",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_execute_tools,
|
||||
):
|
||||
mock_get_tools.return_value = (mock_mcp_tools, ["litellm_proxy"])
|
||||
|
||||
def mock_execute_side_effect(tool_calls, user_api_key_auth, **kwargs):
|
||||
results = []
|
||||
for tool_call in tool_calls:
|
||||
call_id = None
|
||||
if isinstance(tool_call, dict):
|
||||
call_id = tool_call.get("call_id") or tool_call.get("id")
|
||||
elif hasattr(tool_call, "call_id"):
|
||||
call_id = tool_call.call_id
|
||||
elif hasattr(tool_call, "id"):
|
||||
call_id = tool_call.id
|
||||
if call_id:
|
||||
results.append(
|
||||
{
|
||||
"tool_call_id": call_id,
|
||||
"result": "Sunny, 72°F",
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
mock_execute_tools.side_effect = mock_execute_side_effect
|
||||
|
||||
mcp_tool_config = cast(
|
||||
Any,
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_url": "litellm_proxy",
|
||||
"require_approval": "never",
|
||||
},
|
||||
)
|
||||
|
||||
response = await litellm.aresponses(
|
||||
model=model,
|
||||
tools=[mcp_tool_config],
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"type": "message",
|
||||
"content": "What's the weather in San Francisco?",
|
||||
}
|
||||
],
|
||||
stream=True,
|
||||
)
|
||||
|
||||
events = []
|
||||
async for chunk in response:
|
||||
events.append(chunk)
|
||||
|
||||
assert len(events) > 0, "Should receive streaming events"
|
||||
|
||||
created_idx = next(
|
||||
(
|
||||
i
|
||||
for i, e in enumerate(events)
|
||||
if getattr(e, "type", None) == "response.created"
|
||||
),
|
||||
None,
|
||||
)
|
||||
in_progress_idx = next(
|
||||
(
|
||||
i
|
||||
for i, e in enumerate(events)
|
||||
if getattr(e, "type", None) == "response.in_progress"
|
||||
),
|
||||
None,
|
||||
)
|
||||
output_item_added_idx = next(
|
||||
(
|
||||
i
|
||||
for i, e in enumerate(events)
|
||||
if getattr(e, "type", None) == "response.output_item.added"
|
||||
),
|
||||
None,
|
||||
)
|
||||
mcp_in_progress_idx = next(
|
||||
(
|
||||
i
|
||||
for i, e in enumerate(events)
|
||||
if "mcp_list_tools.in_progress" in str(getattr(e, "type", ""))
|
||||
),
|
||||
None,
|
||||
)
|
||||
completed_idx = next(
|
||||
(
|
||||
i
|
||||
for i, e in enumerate(events)
|
||||
if getattr(e, "type", None) == "response.completed"
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
assert created_idx is not None, "response.created event should be present"
|
||||
assert (
|
||||
in_progress_idx is not None
|
||||
), "response.in_progress event should be present"
|
||||
assert (
|
||||
output_item_added_idx is not None
|
||||
), "response.output_item.added event should be present"
|
||||
|
||||
assert (
|
||||
created_idx < in_progress_idx
|
||||
), "response.created should come before response.in_progress"
|
||||
assert (
|
||||
in_progress_idx < output_item_added_idx
|
||||
), "response.in_progress should come before response.output_item.added"
|
||||
|
||||
if mcp_in_progress_idx is not None:
|
||||
assert (
|
||||
output_item_added_idx < mcp_in_progress_idx
|
||||
), "response.output_item.added should come before response.mcp_list_tools.in_progress"
|
||||
|
||||
response_ids = []
|
||||
for i, event in enumerate(events):
|
||||
event_type = getattr(event, "type", None)
|
||||
if hasattr(event, "response"):
|
||||
response_obj = getattr(event, "response", None)
|
||||
if response_obj and hasattr(response_obj, "id"):
|
||||
event_type_value = (
|
||||
event_type.value
|
||||
if hasattr(event_type, "value")
|
||||
else str(event_type)
|
||||
)
|
||||
if any(
|
||||
x in event_type_value
|
||||
for x in [
|
||||
"response.created",
|
||||
"response.in_progress",
|
||||
"response.completed",
|
||||
]
|
||||
):
|
||||
response_ids.append((i, event_type_value, response_obj.id))
|
||||
|
||||
assert (
|
||||
len(response_ids) >= 2
|
||||
), f"Should have at least 2 response lifecycle events. Found {len(response_ids)}"
|
||||
|
||||
cycles = []
|
||||
current_cycle = []
|
||||
current_id = None
|
||||
|
||||
for idx, event_type, resp_id in response_ids:
|
||||
if current_id is None or resp_id == current_id:
|
||||
current_cycle.append((idx, event_type, resp_id))
|
||||
current_id = resp_id
|
||||
else:
|
||||
if current_cycle:
|
||||
cycles.append(current_cycle)
|
||||
current_cycle = [(idx, event_type, resp_id)]
|
||||
current_id = resp_id
|
||||
if current_cycle:
|
||||
cycles.append(current_cycle)
|
||||
|
||||
for cycle_num, cycle in enumerate(cycles):
|
||||
cycle_ids = set(resp_id for _, _, resp_id in cycle)
|
||||
assert (
|
||||
len(cycle_ids) == 1
|
||||
), f"Cycle {cycle_num + 1} should have consistent response ID. Found {len(cycle_ids)} unique IDs"
|
||||
|
||||
assert (
|
||||
completed_idx is not None
|
||||
), "response.completed event should be present"
|
||||
|
||||
lite_errors = [
|
||||
record
|
||||
for record in caplog.records
|
||||
if record.levelno >= logging.ERROR
|
||||
and ("LiteLLM" in record.name or "LiteLLM" in record.getMessage())
|
||||
]
|
||||
assert not lite_errors, "Unexpected LiteLLM errors: " + ", ".join(
|
||||
record.getMessage() for record in lite_errors
|
||||
)
|
||||
|
|
@ -18,8 +18,8 @@ import json
|
|||
async def test_mcp_agent():
|
||||
"""Test MCP agent functionality with a simple math server"""
|
||||
try:
|
||||
local_server_path = "./mcp_server.py"
|
||||
ci_cd_server_path = "tests/mcp_tests/mcp_server.py"
|
||||
local_server_path = "./mcp_math_server.py"
|
||||
ci_cd_server_path = "tests/integration/mcp_sdk1/mcp_math_server.py"
|
||||
|
||||
# Use the correct path for the server
|
||||
server_path = (
|
||||
|
|
|
|||
|
|
@ -0,0 +1,53 @@
|
|||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not os.getenv("GEMINI_API_KEY") and not os.getenv("GOOGLE_API_KEY"),
|
||||
reason="Requires GEMINI_API_KEY or GOOGLE_API_KEY.",
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_gemini_pass_through_endpoint():
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
|
||||
Request,
|
||||
Response,
|
||||
gemini_proxy_route,
|
||||
)
|
||||
|
||||
body = b"""
|
||||
{
|
||||
"contents": [{
|
||||
"parts":[{
|
||||
"text": "The quick brown fox jumps over the lazy dog."
|
||||
}]
|
||||
}]
|
||||
}
|
||||
"""
|
||||
|
||||
# Construct the scope dictionary
|
||||
scope = {
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": "/gemini/v1beta/models/gemini-2.5-flash:countTokens",
|
||||
"query_string": b"key=sk-1234",
|
||||
"headers": [
|
||||
(b"content-type", b"application/json"),
|
||||
],
|
||||
}
|
||||
|
||||
# Create a new Request object
|
||||
async def async_receive():
|
||||
return {"type": "http.request", "body": body, "more_body": False}
|
||||
|
||||
request = Request(
|
||||
scope=scope,
|
||||
receive=async_receive,
|
||||
)
|
||||
|
||||
await gemini_proxy_route(
|
||||
endpoint="v1beta/models/gemini-2.5-flash:countTokens?key=sk-1234",
|
||||
request=request,
|
||||
fastapi_response=Response(),
|
||||
)
|
||||
|
||||
|
|
@ -53,7 +53,8 @@ def client_no_auth():
|
|||
config_fp = (
|
||||
repo_root
|
||||
/ "tests"
|
||||
/ "proxy_unit_tests"
|
||||
/ "unit"
|
||||
/ "proxy"
|
||||
/ "test_configs"
|
||||
/ "test_config_no_auth.yaml"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -340,7 +340,7 @@ def test_a_dockerfile_directory_entry_is_stale_because_only_an_exact_path_exempt
|
|||
def test_a_workflow_that_names_a_file_clears_it_from_the_slice_check():
|
||||
named = coverage._workflow_named_tokens()
|
||||
assert named, "the workflows must name some test paths or the check proves nothing"
|
||||
assert any(coverage._token_covers(token, "tests/local_testing/test_caching_handler.py") for token in named)
|
||||
assert any(coverage._token_covers(token, "tests/proxy_unit_tests/test_proxy_custom_logger.py") for token in named)
|
||||
|
||||
|
||||
def test_the_slice_check_credits_only_workflows_never_the_circleci_config():
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@ CI = [".github/workflows/test-litellm-ui-unit.yml"]
|
|||
),
|
||||
(
|
||||
"cost-map-only",
|
||||
["model_prices_and_context_window.json", "tests/proxy_unit_tests/test_y.py"],
|
||||
["model_prices_and_context_window.json", "tests/unit/proxy/test_y.py"],
|
||||
"run",
|
||||
),
|
||||
(
|
||||
|
|
|
|||
0
tests/unit/enterprise/integrations/__init__.py
Normal file
0
tests/unit/enterprise/integrations/__init__.py
Normal file
0
tests/unit/enterprise/proxy/__init__.py
Normal file
0
tests/unit/enterprise/proxy/__init__.py
Normal file
0
tests/unit/enterprise/proxy/auth/__init__.py
Normal file
0
tests/unit/enterprise/proxy/auth/__init__.py
Normal file
0
tests/unit/enterprise/proxy/guardrails/__init__.py
Normal file
0
tests/unit/enterprise/proxy/guardrails/__init__.py
Normal file
0
tests/unit/enterprise/proxy/hooks/__init__.py
Normal file
0
tests/unit/enterprise/proxy/hooks/__init__.py
Normal file
0
tests/unit/gateway/__init__.py
Normal file
0
tests/unit/gateway/__init__.py
Normal file
0
tests/unit/litellm_proxy_extras/__init__.py
Normal file
0
tests/unit/litellm_proxy_extras/__init__.py
Normal file
|
|
@ -9,7 +9,7 @@ import pytest
|
|||
sys.path.insert(
|
||||
0,
|
||||
os.path.abspath(
|
||||
os.path.join(os.path.dirname(__file__), "../../litellm-proxy-extras")
|
||||
os.path.join(os.path.dirname(__file__), "../../../litellm-proxy-extras")
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -23,7 +23,7 @@ from litellm_proxy_extras.utils import (
|
|||
_MIGRATIONS_DIR = os.path.abspath(
|
||||
os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"../../litellm-proxy-extras/litellm_proxy_extras/migrations",
|
||||
"../../../litellm-proxy-extras/litellm_proxy_extras/migrations",
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -999,7 +999,7 @@ class TestJWTKeyMappingCascade:
|
|||
schema_paths = glob.glob(
|
||||
os.path.abspath(
|
||||
os.path.join(
|
||||
os.path.dirname(__file__), "../../**/schema.prisma"
|
||||
os.path.dirname(__file__), "../../../**/schema.prisma"
|
||||
)
|
||||
),
|
||||
recursive=True,
|
||||
0
tests/unit/proxy/__init__.py
Normal file
0
tests/unit/proxy/__init__.py
Normal file
0
tests/unit/proxy/_experimental/__init__.py
Normal file
0
tests/unit/proxy/_experimental/__init__.py
Normal file
0
tests/unit/proxy/_experimental/mcp_server/__init__.py
Normal file
0
tests/unit/proxy/_experimental/mcp_server/__init__.py
Normal file
78
tests/unit/proxy/_experimental/mcp_server/conftest.py
Normal file
78
tests/unit/proxy/_experimental/mcp_server/conftest.py
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
import asyncio
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def event_loop():
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
yield loop
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
def setup_and_teardown():
|
||||
"""
|
||||
This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained.
|
||||
"""
|
||||
importlib.reload(litellm)
|
||||
import asyncio
|
||||
|
||||
loop = asyncio.get_event_loop_policy().new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
yield
|
||||
|
||||
# Teardown code (executes after the yield point)
|
||||
# LoggingWorker carries still-queued coroutines onto the next test's loop, where they'd log into that test's callbacks
|
||||
asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue())
|
||||
loop.close() # Close the loop created earlier
|
||||
asyncio.set_event_loop(None) # Remove the reference to the loop
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
async def drain_logging_worker():
|
||||
"""
|
||||
The logging queue is bound to the running loop, so anything left queued when a test's loop
|
||||
goes away is carried onto the next test's loop and fires against its callbacks.
|
||||
"""
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
yield
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.clear_queue(), timeout=10)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config, items):
|
||||
# Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests
|
||||
custom_logger_tests = [
|
||||
item for item in items if "custom_logger" in item.parent.name
|
||||
]
|
||||
other_tests = [item for item in items if "custom_logger" not in item.parent.name]
|
||||
|
||||
# Sort tests based on their names
|
||||
custom_logger_tests.sort(key=lambda x: x.name)
|
||||
other_tests.sort(key=lambda x: x.name)
|
||||
|
||||
# Reorder the items list
|
||||
items[:] = custom_logger_tests + other_tests
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def config_only_mcp_manager_factory():
|
||||
from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager
|
||||
|
||||
class ConfigOnlyManager(MCPServerManager):
|
||||
def initialize_tool_name_to_mcp_server_name_mapping(self):
|
||||
return None
|
||||
|
||||
return ConfigOnlyManager
|
||||
0
tests/unit/proxy/auth/__init__.py
Normal file
0
tests/unit/proxy/auth/__init__.py
Normal file
0
tests/unit/proxy/common_utils/__init__.py
Normal file
0
tests/unit/proxy/common_utils/__init__.py
Normal file
150
tests/unit/proxy/conftest.py
Normal file
150
tests/unit/proxy/conftest.py
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
# conftest.py
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import inspect
|
||||
import warnings
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
import litellm
|
||||
import litellm.proxy.proxy_server
|
||||
|
||||
|
||||
# Top-level assignments of these types are the ones importlib.reload(litellm)
|
||||
# would have effectively reset. We snapshot them at conftest import time and
|
||||
# deep-copy the snapshot back before every test.
|
||||
_SNAPSHOT_TYPES = (list, dict, set, tuple, str, int, float, bool, bytes)
|
||||
|
||||
|
||||
def _snapshot_mutable_state(module):
|
||||
"""Capture a per-module snapshot of primitive and collection attributes."""
|
||||
snapshot = {}
|
||||
for attr in list(vars(module)):
|
||||
if attr.startswith("_"):
|
||||
continue
|
||||
try:
|
||||
value = getattr(module, attr)
|
||||
except Exception as exc:
|
||||
warnings.warn(
|
||||
f"conftest: could not read {module.__name__}.{attr} during snapshot: {exc}",
|
||||
stacklevel=2,
|
||||
)
|
||||
continue
|
||||
if value is None or isinstance(value, _SNAPSHOT_TYPES):
|
||||
try:
|
||||
snapshot[attr] = copy.deepcopy(value)
|
||||
except Exception as exc:
|
||||
warnings.warn(
|
||||
f"conftest: could not snapshot {module.__name__}.{attr}: {exc}",
|
||||
stacklevel=2,
|
||||
)
|
||||
return snapshot
|
||||
|
||||
|
||||
def _restore_mutable_state(module, snapshot):
|
||||
for attr, default in snapshot.items():
|
||||
try:
|
||||
setattr(module, attr, copy.deepcopy(default))
|
||||
except Exception as exc:
|
||||
warnings.warn(
|
||||
f"conftest: could not restore {module.__name__}.{attr}: {exc}",
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
|
||||
def _collect_flushable_caches():
|
||||
"""Return (module, attr) pairs whose values expose flush_cache()."""
|
||||
targets = []
|
||||
for module in (litellm, litellm.proxy.proxy_server):
|
||||
for attr in list(vars(module)):
|
||||
if attr.startswith("_"):
|
||||
continue
|
||||
try:
|
||||
value = getattr(module, attr)
|
||||
except Exception:
|
||||
continue
|
||||
# Only instances — a class reference has an unbound flush_cache
|
||||
# that can't be called without a self argument.
|
||||
if inspect.isclass(value) or inspect.ismodule(value):
|
||||
continue
|
||||
if callable(getattr(value, "flush_cache", None)):
|
||||
targets.append((module, attr))
|
||||
return targets
|
||||
|
||||
|
||||
def _flush_caches(targets):
|
||||
for module, attr in targets:
|
||||
try:
|
||||
value = getattr(module, attr)
|
||||
except Exception:
|
||||
continue
|
||||
flush = getattr(value, "flush_cache", None)
|
||||
if callable(flush):
|
||||
try:
|
||||
flush()
|
||||
except Exception as exc:
|
||||
warnings.warn(
|
||||
f"conftest: flush_cache failed on {module.__name__}.{attr}: {exc}",
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
|
||||
# Snapshot once at conftest import — these are the "clean" module states.
|
||||
_LITELLM_STATE = _snapshot_mutable_state(litellm)
|
||||
_PROXY_SERVER_STATE = _snapshot_mutable_state(litellm.proxy.proxy_server)
|
||||
_FLUSHABLE_CACHES = _collect_flushable_caches()
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
def setup_and_teardown():
|
||||
"""Reset mutable module state on litellm and proxy_server before each test.
|
||||
|
||||
Replaces a previous importlib.reload(litellm) approach that cost ~17s
|
||||
per test (re-executing the full litellm __init__ import chain).
|
||||
|
||||
What IS reset:
|
||||
- Top-level module attributes of type list / dict / set / tuple
|
||||
/ str / int / float / bool / bytes, and None-valued attributes.
|
||||
These cover callback lists, general_settings, master_key,
|
||||
premium_user, prisma_client, etc. — anything the old reload() reset
|
||||
by re-executing the module body.
|
||||
- Any module-level object instance that exposes flush_cache() (the
|
||||
DualCache and LLMClientCache family), which handles cache state
|
||||
that can't round-trip through deepcopy because of internal locks.
|
||||
|
||||
What is NOT reset:
|
||||
- Class instances without flush_cache() (e.g. ProxyLogging,
|
||||
JWTHandler, FastAPI routers, loggers). If a test mutates such an
|
||||
instance in-place (setattr on the instance, appending to one of
|
||||
its internal lists, etc.), the mutation will leak into later tests.
|
||||
Use pytest's monkeypatch.setattr() or a local fixture for those
|
||||
cases — don't rely on this autouse fixture to undo them.
|
||||
"""
|
||||
_restore_mutable_state(litellm, _LITELLM_STATE)
|
||||
_restore_mutable_state(litellm.proxy.proxy_server, _PROXY_SERVER_STATE)
|
||||
_flush_caches(_FLUSHABLE_CACHES)
|
||||
|
||||
loop = asyncio.get_event_loop_policy().new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
loop.close()
|
||||
asyncio.set_event_loop(None)
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config, items):
|
||||
# Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests
|
||||
custom_logger_tests = [
|
||||
item for item in items if "custom_logger" in item.parent.name
|
||||
]
|
||||
other_tests = [item for item in items if "custom_logger" not in item.parent.name]
|
||||
|
||||
# Sort tests based on their names
|
||||
custom_logger_tests.sort(key=lambda x: x.name)
|
||||
other_tests.sort(key=lambda x: x.name)
|
||||
|
||||
# Reorder the items list
|
||||
items[:] = custom_logger_tests + other_tests
|
||||
0
tests/unit/proxy/db/__init__.py
Normal file
0
tests/unit/proxy/db/__init__.py
Normal file
0
tests/unit/proxy/db/db_transaction_queue/__init__.py
Normal file
0
tests/unit/proxy/db/db_transaction_queue/__init__.py
Normal file
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue