diff --git a/.circleci/config.yml b/.circleci/config.yml index e4333c88a64..40ed959ff29 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,7 +38,7 @@ commands: parameters: category: type: enum - enum: ["backend", "client", "provider-harness", "redis-compat"] + enum: ["backend", "client", "redis-compat"] default: "backend" steps: - run: @@ -619,7 +619,7 @@ jobs: name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) command: | mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") + TEST_FILES=$(circleci tests glob "tests/integration/configuration/test_basic_proxy_startup.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ @@ -696,7 +696,7 @@ jobs: name: Run second round of tests command: | mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") + TEST_FILES=$(circleci tests glob "tests/integration/configuration/test_basic_proxy_startup.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ @@ -708,84 +708,46 @@ jobs: # Store test results - store_test_results: path: test-results - proxy_pass_through_endpoint_tests: - machine: - image: ubuntu-2204:2024.04.1 + e2e_ui_build: + docker: + - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} resource_class: large working_directory: ~/project steps: - checkout - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - skip_if_unrelated_changes + - skip_if_unrelated_changes: + category: client - setup_google_dns - - install_uv - - install_rust - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_postgres - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Run Docker container - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -e GEMINI_API_KEY=$GEMINI_API_KEY \ - -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ - -e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \ - -e AZURE_API_KEY=$AZURE_API_KEY \ - -e AZURE_API_BASE=$AZURE_API_BASE \ - -e USE_DDTRACE=True \ - -e DD_API_KEY=$DD_API_KEY \ - -e DD_SITE=$DD_SITE \ - -e LITELLM_LICENSE=$LITELLM_LICENSE \ - -e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \ - -e LITELLM_LOG=ERROR \ - --add-host host.docker.internal:host-gateway \ - --name my-app \ - -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \ - -v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 - - run: - name: Start outputting logs - command: docker logs -f my-app - background: true - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - # Install Node.js directly from nodejs.org with SHA256 verification, - # instead of piping NodeSource's setup_24.x apt-repo installer into - # sudo bash (which runs a mutable upstream script unattended). - install_node - + - restore_cache: + keys: + - ui-dashboard-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - run: - name: Install Node.js test dependencies + name: Install dashboard dependencies command: | - cd tests/pass_through_tests + cd ui/litellm-dashboard npm ci - + - save_cache: + key: ui-dashboard-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/node_modules - run: - name: Run Vertex AI, Google AI Studio Node.js tests + name: Build UI bundle command: | - cd tests/pass_through_tests - NODE_OPTIONS=--experimental-vm-modules npx jest . --verbose - no_output_timeout: 30m + cd ui/litellm-dashboard + npm run build + mkdir -p ../../ui-bundle + mv out ../../ui-bundle/out + find ../../ui-bundle/out -name '*.html' ! -name 'index.html' | while read -r f; do + d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" + done + - persist_to_workspace: + root: ~/project + paths: + - ui-bundle/out e2e_ui_testing: docker: @@ -798,7 +760,7 @@ jobs: POSTGRES_USER: e2euser POSTGRES_PASSWORD: e2epassword POSTGRES_DB: litellm_e2e - resource_class: large + resource_class: xlarge working_directory: ~/project environment: DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" @@ -817,6 +779,8 @@ jobs: printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - skip_if_unrelated_changes: category: client + - attach_workspace: + at: ~/project - setup_google_dns - install_node - install_uv @@ -835,7 +799,7 @@ jobs: - ~/.cache/uv - restore_cache: keys: - - ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} + - ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - run: name: Install Node dependencies and Playwright # The cimg/python:3.12-browsers image already ships the Chromium system @@ -844,36 +808,19 @@ jobs: # 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors, # so we install just the browser binary. command: | - cd ui/litellm-dashboard - npm ci - cd ../../tests/e2e/ui + cd tests/e2e/ui npm ci npx playwright install chromium - save_cache: - key: ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} + key: ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} paths: - - ui/litellm-dashboard/node_modules - tests/e2e/ui/node_modules - ~/.cache/ms-playwright - run: - name: Build UI from source - # Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`. - # GNU cp (used on CircleCI's Ubuntu image) interprets that as "copy the - # source directory as a child of the destination" when the destination - # already exists — silently creating `_experimental/out/out/` instead of - # replacing the served bundle. The proxy continued serving whatever was - # checked into `_experimental/out/*`, so this job was effectively testing - # the pre-build bundle on every run. Replace-and-move guarantees the - # freshly built bundle is what the proxy actually serves. + name: Use built UI bundle command: | - cd ui/litellm-dashboard - npm run build - rm -rf ../../litellm/proxy/_experimental/out - mv out ../../litellm/proxy/_experimental/out - # Restructure HTML so extensionless routes work (login.html -> login/index.html) - find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do - d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" - done + rm -rf litellm/proxy/_experimental/out + cp -r ui-bundle/out litellm/proxy/_experimental/out - wait_for_service: url: tcp://localhost:5432 timeout: "30" @@ -940,7 +887,7 @@ jobs: command: | cd tests/e2e/ui LITELLM_LICENSE="$LITELLM_LICENSE" \ - npx playwright test --config playwright.config.ts + npx playwright test --config playwright.config.ts --workers=3 --retries=0 no_output_timeout: 10m - store_artifacts: path: tests/e2e/ui/test-results @@ -978,6 +925,8 @@ jobs: printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - skip_if_unrelated_changes: category: client + - attach_workspace: + at: ~/project - setup_google_dns - install_node - install_uv @@ -996,31 +945,23 @@ jobs: - ~/.cache/uv - restore_cache: keys: - - ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} + - ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - run: name: Install Node dependencies and Playwright command: | - cd ui/litellm-dashboard - npm ci - cd ../../tests/e2e/ui + cd tests/e2e/ui npm ci npx playwright install chromium - save_cache: - key: ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} + key: ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} paths: - - ui/litellm-dashboard/node_modules - tests/e2e/ui/node_modules - ~/.cache/ms-playwright - run: - name: Build UI from source + name: Use built UI bundle command: | - cd ui/litellm-dashboard - npm run build - rm -rf ../../litellm/proxy/_experimental/out - mv out ../../litellm/proxy/_experimental/out - find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do - d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" - done + rm -rf litellm/proxy/_experimental/out + cp -r ui-bundle/out litellm/proxy/_experimental/out - wait_for_service: url: tcp://localhost:5432 timeout: "30" @@ -1271,45 +1212,6 @@ jobs: ! docker exec my-app true 2>/dev/null || fail "container is still running" echo "Proxy refused to serve (exit $exit_code) and never reached startup. Test passed." - provider_replay_harness: - docker: - - &python312_image - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - - image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f - working_directory: ~/project - resource_class: medium - environment: - E2E_CACHE_TEST_REDIS_URL: redis://127.0.0.1:6379/0 - E2E_PROVIDER_CACHE: "0" - E2E_FIXTURE_MODE: live - steps: - - checkout - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - skip_if_unrelated_changes: - category: provider-harness - - setup_litellm_test_deps - - wait_for_service: - url: tcp://localhost:6379 - - run: - name: Test provider capture and replay harness - command: | - mkdir -p test-results/provider-replay-harness - uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures \ - --junitxml=test-results/provider-replay-harness/junit.xml \ - tests/e2e_harness/test_provider_edge.py tests/e2e_harness/test_fixture_bundle.py \ - tests/e2e_harness/test_fixture_canonical.py tests/e2e_harness/test_fixture_mode.py \ - tests/code_coverage_tests/test_provider_replay_harness.py \ - tests/code_coverage_tests/test_provider_cache.py - - store_test_results: - path: test-results/provider-replay-harness - integration_contracts: parameters: suite: @@ -1682,9 +1584,14 @@ jobs: path: test-results workflows: - migration_startup: - when: << pipeline.parameters.run_migration_tests >> - jobs: &migration_jobs + migration_startup_scheduled: + triggers: + - schedule: + cron: "17 0,6,12,18 * * *" + filters: + branches: + only: main + jobs: - build_docker_database_image: migration_qualification: true - migration_startup_tests: @@ -1709,33 +1616,11 @@ workflows: suite: shaped baseline: true requires: [build_docker_database_image] - migration_startup_scheduled: - triggers: - - schedule: - cron: "17 0,6,12,18 * * *" - filters: - branches: - only: main - jobs: *migration_jobs - routing_parity: - when: - not: - equal: ["", << pipeline.parameters.routing_parity_base >>] - jobs: - - routing_parity: - name: routing-parity-<< matrix.suite >> - matrix: - parameters: - suite: [management, accounting, database, providers, extensions, cost, mcp] integration: - unless: - or: - - << pipeline.parameters.run_migration_tests >> - - not: - equal: ["", << pipeline.parameters.routing_parity_base >>] jobs: - integration_contracts: name: integration-<< matrix.suite >> + filters: &integration_default_run not pipeline.parameters.run_migration_tests and pipeline.parameters.routing_parity_base == "" matrix: parameters: suite: [sdk, cost, browser] @@ -1745,116 +1630,131 @@ workflows: suite: security parallelism: 1 resource_class: xlarge + filters: *integration_default_run - integration_contracts: name: integration-management suite: management parallelism: 4 resource_class: xlarge + filters: *integration_default_run - integration_shard_coverage: name: integration-management-coverage suite: management shards: 4 requires: [integration-management] + filters: *integration_default_run - integration_contracts: name: integration-accounting suite: accounting parallelism: 4 resource_class: xlarge + filters: *integration_default_run - integration_shard_coverage: name: integration-accounting-coverage suite: accounting shards: 4 requires: [integration-accounting] + filters: *integration_default_run - integration_contracts: name: integration-providers suite: providers parallelism: 4 resource_class: xlarge + filters: *integration_default_run - integration_shard_coverage: name: integration-providers-coverage suite: providers shards: 4 requires: - integration-providers + filters: *integration_default_run - integration_contracts: name: integration-database suite: database parallelism: 2 + filters: *integration_default_run - integration_shard_coverage: name: integration-database-coverage suite: database shards: 2 requires: [integration-database] + filters: *integration_default_run - integration_contracts: name: integration-mcp suite: mcp resource_class: xlarge parallelism: 2 + filters: *integration_default_run - integration_shard_coverage: name: integration-mcp-coverage suite: mcp shards: 2 requires: - integration-mcp + filters: *integration_default_run - integration_contracts: name: integration-extensions suite: extensions parallelism: 8 resource_class: xlarge + filters: *integration_default_run - integration_shard_coverage: name: integration-extensions-coverage suite: extensions shards: 8 requires: - integration-extensions + filters: *integration_default_run - integration_contracts: name: integration-management-replica suite: management mode: replica parallelism: 4 resource_class: xlarge + filters: *integration_default_run - integration_shard_coverage: name: integration-management-replica-coverage suite: management mode: replica shards: 4 requires: [integration-management-replica] + filters: *integration_default_run - integration_contracts: name: integration-database-replica suite: database mode: replica parallelism: 2 + filters: *integration_default_run - integration_shard_coverage: name: integration-database-replica-coverage suite: database mode: replica shards: 2 requires: [integration-database-replica] - - postgres_suite: - name: proxy-behavior - test_path: tests/proxy_behavior - seed: true - coverage_flag: lens-postgres - timeout_minutes: 25 + filters: *integration_default_run - postgres_suite: name: proxy-security test_path: tests/proxy_security_tests seed: true timeout_minutes: 15 + filters: *integration_default_run - postgres_suite: name: schema-migration test_path: tests/proxy_migration_tests seed: false timeout_minutes: 20 workers: 2 + filters: *integration_default_run - postgres_suite: name: roi-database test_path: tests/integration/database/test_roi_observed.py seed: false coverage_flag: roi-postgres timeout_minutes: 10 + filters: *integration_default_run - mcp_integration: name: mcp-integration + filters: *integration_default_run - redis_compat: name: redis-compat-<< matrix.redis_py >> matrix: @@ -1864,31 +1764,71 @@ workflows: - "6.4.0" - "7.4.1" - "8.0.1" - - windows_release_wheel - - base_sdk_install - build_and_test: - unless: - or: - - << pipeline.parameters.run_migration_tests >> - - not: - equal: ["", << pipeline.parameters.routing_parity_base >>] - jobs: - - provider_replay_harness - - build_docker_database_image - - e2e_ui_testing - - e2e_ui_testing_server_root_path + filters: *integration_default_run + - windows_release_wheel: + filters: *integration_default_run + - base_sdk_install: + filters: *integration_default_run + - build_docker_database_image: + filters: *integration_default_run - proxy_logging_guardrails_model_info_tests: requires: - build_docker_database_image - - proxy_pass_through_endpoint_tests: - requires: - - build_docker_database_image + filters: *integration_default_run - db_migration_disable_update_check: requires: - build_docker_database_image - - helm_chart_testing: - requires: - - build_docker_database_image + filters: *integration_default_run - test_bad_database_url: requires: - build_docker_database_image + filters: *integration_default_run + - helm_chart_testing: + requires: + - build_docker_database_image + filters: *integration_default_run + - e2e_ui_build: + filters: *integration_default_run + - e2e_ui_testing: + requires: [e2e_ui_build] + filters: *integration_default_run + - e2e_ui_testing_server_root_path: + requires: [e2e_ui_build] + filters: *integration_default_run + - build_docker_database_image: + name: build_docker_database_image-migration + migration_qualification: true + filters: pipeline.parameters.run_migration_tests + - migration_startup_tests: + name: migration-startup + suite: startup + requires: [build_docker_database_image-migration] + filters: pipeline.parameters.run_migration_tests + - migration_startup_tests: + name: migration-recovery + suite: recovery + requires: [build_docker_database_image-migration] + filters: pipeline.parameters.run_migration_tests + - migration_startup_tests: + name: migration-legacy-and-pooling + suite: legacy + requires: [build_docker_database_image-migration] + filters: pipeline.parameters.run_migration_tests + - migration_startup_tests: + name: migration-upgrade + suite: upgrade + baseline: true + requires: [build_docker_database_image-migration] + filters: pipeline.parameters.run_migration_tests + - migration_startup_tests: + name: migration-upgrade-shaped + suite: shaped + baseline: true + requires: [build_docker_database_image-migration] + filters: pipeline.parameters.run_migration_tests + - routing_parity: + name: routing-parity-<< matrix.suite >> + matrix: + parameters: + suite: [management, accounting, database, providers, extensions, cost, mcp] + filters: pipeline.parameters.routing_parity_base != "" diff --git a/.circleci/scripts/classify_changes.sh b/.circleci/scripts/classify_changes.sh index 9fe2e99fb93..cca7f19a02a 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.circleci/scripts/classify_changes.sh @@ -1,12 +1,11 @@ #!/usr/bin/env bash set -uo pipefail -category="${1:?usage: classify_changes.sh }" +category="${1:?usage: classify_changes.sh }" has_client=false has_backend=false has_ci=false -has_provider_harness=false has_cost_map=false has_mcp_dependencies=false has_windows_release=false @@ -23,11 +22,6 @@ while IFS= read -r file || [ -n "$file" ]; do pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .circleci/scripts/classify_changes.sh | tests/unit/test_circleci_path_filter.py | tests/unit/test_detect_changes.py) has_mcp_dependencies=true ;; esac - case "$file" in - tests/e2e/*/*.py | tests/e2e_harness/*/*.py) : ;; - tests/e2e/*.py | tests/e2e_harness/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/unit/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock) - has_provider_harness=true ;; - esac case "$file" in litellm-rust/* | litellm/rust_bridge/* | rust-toolchain.toml | pyproject.toml | uv.lock | tests/windows_tests/* | .circleci/*) has_windows_release=true ;; @@ -41,7 +35,7 @@ while IFS= read -r file || [ -n "$file" ]; do case "$file" in model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json) has_cost_map=true ;; - tests/test_litellm/* | tests/proxy_unit_tests/* | tests/unit/proxy/*) : ;; + tests/unit/proxy/*) : ;; *) outside_cost_map_set=true ;; esac done @@ -53,9 +47,6 @@ case "$category" in cost-map-only) { [ "$has_cost_map" = true ] && [ "$outside_cost_map_set" = false ]; } && echo run || echo skip ;; - provider-harness) - [ "$has_provider_harness" = true ] && echo run || echo skip - ;; windows-release) [ "$has_windows_release" = true ] && echo run || echo skip ;; diff --git a/.circleci/scripts/path_filter.sh b/.circleci/scripts/path_filter.sh index ccc68a0750b..111fa2b57ca 100755 --- a/.circleci/scripts/path_filter.sh +++ b/.circleci/scripts/path_filter.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -uo pipefail -category="${1:?usage: path_filter.sh }" +category="${1:?usage: path_filter.sh }" here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" run_full() { diff --git a/.circleci/scripts/run_migration_tests.py b/.circleci/scripts/run_migration_tests.py index 5a73c54e3f5..50d01f1d50b 100644 --- a/.circleci/scripts/run_migration_tests.py +++ b/.circleci/scripts/run_migration_tests.py @@ -10,7 +10,7 @@ from typing import Final from xml.etree import ElementTree SUITES: Final = { - "startup": (("test_startup.py",), 12), + "startup": (("test_startup.py",), 13), "recovery": (("test_recovery.py",), 15), "legacy": (("test_legacy.py", "test_pooling.py"), 11), "upgrade": (("test_upgrade.py", "test_rolling_upgrade.py"), 5), diff --git a/.github/ci-coverage-allowlist.yml b/.github/ci-coverage-allowlist.yml index 85c8dcb1ab2..db2f652b877 100644 --- a/.github/ci-coverage-allowlist.yml +++ b/.github/ci-coverage-allowlist.yml @@ -23,18 +23,6 @@ test_paths: and that pull request gives it a new home paths: - tests/test_rust_python_harness.py - - reason: >- - Live-provider caching cases in tests/local_testing that remain outside CI. No CircleCI job - globs that directory anymore, and the legacy jobs that did always deselected these files. - The two Redis Cluster IAM authentication tests moved to tests/integration/sdk and remain - selected by node ID in redis_compat, where redis-server is installed. - The five keyless files now run in the caching-local shard, leaving live cases in these two. - Measured 2026-08-21 with no provider credentials: test_disk_cache_unit_tests.py needs - OPENAI_API_KEY for 2 of its 4 tests, and test_gcs_cache_unit_tests.py needs GCS credentials - for all 4. Keep these live cases outside PR CI rather than making a job red by construction - paths: - - tests/local_testing/test_disk_cache_unit_tests.py - - tests/local_testing/test_gcs_cache_unit_tests.py - reason: >- The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than from a pull request; it needs a live gateway and provider credentials no PR job holds @@ -60,48 +48,6 @@ test_paths: choice it informed is settled paths: - tests/code_coverage_tests/test_aio_http_image_conversion.py - - reason: >- - No job invokes this suite and its files mix pure transformation tests with ones driving live - vendor vector stores, so assigning them needs a per-file decision - paths: - - tests/vector_store_tests/rag/test_rag_bedrock.py - - tests/vector_store_tests/rag/test_rag_openai.py - - tests/vector_store_tests/rag/test_rag_s3_vectors.py - - tests/vector_store_tests/rag/test_rag_vertex_ai.py - - tests/vector_store_tests/test_azure_ai_vector_store.py - - tests/vector_store_tests/test_azure_vector_store.py - - tests/vector_store_tests/test_bedrock_vector_store.py - - tests/vector_store_tests/test_gemini_vector_store.py - - tests/vector_store_tests/test_milvus_vector_store.py - - tests/vector_store_tests/test_openai_vector_store.py - - tests/vector_store_tests/test_ragflow_vector_store.py - - tests/vector_store_tests/test_s3_vectors_vector_store.py - - tests/vector_store_tests/test_vertex_ai_search_api_vector_store.py - - tests/vector_store_tests/test_vertex_ai_vector_store.py - - reason: >- - Throughput and memory-growth measurements whose runtime and variance make them unsuitable for - a per-pull-request job - paths: - - tests/load_tests/test_datadog_load_test.py - - tests/load_tests/test_langsmith_load_test.py - - tests/load_tests/test_linear_memory_growth.py - - tests/load_tests/test_memory_usage.py - - tests/load_tests/test_otel_load_test.py - - tests/load_tests/test_vertex_embeddings_load_test.py - - tests/load_tests/test_vertex_load_tests.py - - reason: >- - Env-gated saturation benchmark requires a live proxy and provider credentials, so it is run - locally rather than in pull-request jobs - paths: - - tests/load_tests/test_granian_admission_saturation.py - - reason: >- - A local-only agent rig: test_a2a_completion_bridge.py needs a LangGraph server on - localhost:2024 and test_a2a.py drives a live A2A endpoint, so neither can run in a - pull request job. Until 2026-08-20 the CircleCI agent job hid them behind a grep -v - that this census could not see; the glob now excludes them structurally and this entry - is the decision on the record. Revisit when the A2A bridge gets a recorded-wire fixture - paths: - - tests/agent_tests/local_only_agent_tests - reason: >- Third-party integration tests that skip themselves without OCI configuration or sandbox credentials, neither of which a pull request job holds diff --git a/.github/scripts/assert_ci_coverage.py b/.github/scripts/assert_ci_coverage.py index 5c581c99d12..fbdbfafa66d 100644 --- a/.github/scripts/assert_ci_coverage.py +++ b/.github/scripts/assert_ci_coverage.py @@ -37,7 +37,6 @@ GLOB_CHARS = frozenset("*?") # tests has to be named by some shard or it runs nowhere. A child listed here is # itself decomposed one level deeper and is checked through its own entry. SHARDED_ROOTS: tuple[str, ...] = ( - "tests/test_litellm", "tests/unit", "tests/unit/enterprise", "tests/unit/enterprise/enterprise_callbacks", diff --git a/.github/workflows/deploy-dev-on-main.yml b/.github/workflows/deploy-dev-on-main.yml new file mode 100644 index 00000000000..1208a2dbbdb --- /dev/null +++ b/.github/workflows/deploy-dev-on-main.yml @@ -0,0 +1,30 @@ +name: Deploy main to gateway-dev + +on: + push: + branches: [main] + +permissions: {} + +concurrency: + group: deploy-dev-dispatch + cancel-in-progress: true + +jobs: + dispatch: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2.2.2 + id: token + with: + app-id: ${{ secrets.LITELLM_OPS_DISPATCH_APP_ID }} + private-key: ${{ secrets.LITELLM_OPS_DISPATCH_APP_PRIVATE_KEY }} + owner: BerriAI + repositories: litellm-ops + permission-actions: write + - name: Dispatch the litellm-ops dev deploy + env: + GH_TOKEN: ${{ steps.token.outputs.token }} + run: gh api --method POST repos/BerriAI/litellm-ops/actions/workflows/deploy-dev.yml/dispatches -f ref=main diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 2d0766e0468..5384e873f22 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -22,6 +22,16 @@ jobs: actions: read runs-on: ubuntu-latest timeout-minutes: 15 + services: + redis: + image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: @@ -189,8 +199,9 @@ jobs: if: steps.changes.outputs.decision != 'skip' env: LITELLM_MASTER_KEY: sk-e2e-harness-tests-reach-no-proxy + E2E_CACHE_TEST_REDIS_URL: redis://127.0.0.1:6379/0 run: | - if ! git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- tests/e2e tests/e2e_harness ':(exclude)tests/e2e/ui' pyproject.toml uv.lock .github/workflows/test-linting.yml | grep -q .; then + if ! git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- tests/e2e tests/e2e_harness ':(exclude)tests/e2e/ui' tests/code_coverage_tests/test_provider_replay_harness.py tests/code_coverage_tests/test_provider_cache.py pyproject.toml uv.lock .github/workflows/test-linting.yml | grep -q .; then echo "No changed e2e harness files; skipping." exit 0 fi @@ -198,6 +209,9 @@ jobs: CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)" tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli" PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q tests/e2e_harness + uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures \ + tests/code_coverage_tests/test_provider_replay_harness.py \ + tests/code_coverage_tests/test_provider_cache.py - name: Check for circular imports if: steps.changes.outputs.decision != 'skip' diff --git a/.github/workflows/test-merge-smoke.yml b/.github/workflows/test-merge-smoke.yml index 0133629d24a..0d1cc6596de 100644 --- a/.github/workflows/test-merge-smoke.yml +++ b/.github/workflows/test-merge-smoke.yml @@ -234,7 +234,30 @@ jobs: exit 1 - name: Run the startup tests - run: .venv/bin/python -m pytest --tb=short -vv tests/basic_proxy_startup_tests + shell: bash + run: | + set -euo pipefail + base_url=http://localhost:4000 + readiness_response="$(curl -sS --fail-with-body --write-out $'\n%{http_code}' "${base_url}/health/readiness")" + readiness_status="${readiness_response##*$'\n'}" + readiness_body="${readiness_response%$'\n'*}" + [[ "${readiness_status}" == "200" ]] + jq -e '.status == "healthy" or .status == "connected"' <<< "${readiness_body}" > /dev/null + liveness_status="$(curl -sS --fail-with-body --output /dev/null --write-out '%{http_code}' "${base_url}/health/liveness")" + [[ "${liveness_status}" == "200" ]] + completion_response="$(curl -sS --fail-with-body --write-out $'\n%{http_code}' \ + --header "Authorization: Bearer ${LITELLM_MASTER_KEY}" \ + --header 'Content-Type: application/json' \ + --data '{"model":"gpt-4","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Hello!"}]}' \ + "${base_url}/chat/completions")" + completion_status="${completion_response##*$'\n'}" + completion_body="${completion_response%$'\n'*}" + [[ "${completion_status}" == "200" ]] + jq -e ' + (.choices | length) == 1 and + .choices[0].message.role == "assistant" and + .choices[0].message.content == "Hello! This is a mock response from the fake OpenAI endpoint." + ' <<< "${completion_body}" > /dev/null - name: Show the proxy logs if: always() diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index fe1d2c025b3..f8a7c38ef79 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -18,7 +18,6 @@ on: - "litellm/ocr/**" - "litellm/llms/base_llm/ocr/**" - "litellm/llms/custom_httpx/llm_http_handler.py" - - "tests/test_litellm/conftest.py" - "Makefile" - ".cargo/**" - "pyproject.toml" @@ -49,7 +48,6 @@ on: - "litellm/ocr/**" - "litellm/llms/base_llm/ocr/**" - "litellm/llms/custom_httpx/llm_http_handler.py" - - "tests/test_litellm/conftest.py" - "Makefile" - ".cargo/**" - "pyproject.toml" diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 1dc9db0a0c7..23a340cd436 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -174,8 +174,6 @@ jobs: - shard: integrations test-path: |- - tests/test_litellm/integrations - tests/test_litellm/tracing tests/unit/integrations python-version: "3.12" workers: 4 @@ -292,6 +290,7 @@ jobs: tests/unit/llms/manus tests/unit/llms/meta_llama tests/unit/llms/microsoft_365_copilot + tests/unit/llms/milvus tests/unit/llms/minimax tests/unit/llms/mistral tests/unit/llms/modelscope @@ -376,7 +375,6 @@ jobs: - shard: root test-path: >- - tests/test_litellm/test_*.py tests/unit/test_*.py python-version: "3.12" workers: 4 @@ -395,16 +393,6 @@ jobs: job-timeout-minutes: 60 dist: loadscope - - shard: router-unit-tests - test-path: >- - tests/router_unit_tests - python-version: "3.12" - workers: 0 - reruns: 0 - timeout-minutes: 10 - job-timeout-minutes: 60 - dist: loadscope - - shard: endpoints test-path: >- tests/unit/a2a_protocol @@ -775,7 +763,7 @@ jobs: - shard: proxy-server-core test-path: |- - tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py + tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py tests/unit/proxy/test__lazy_features.py tests/unit/proxy/test_aproxy_startup.py tests/unit/proxy/test_proxy_server.py @@ -816,7 +804,7 @@ jobs: - shard: custom-logging test-path: |- - tests/proxy_unit_tests/test_proxy_custom_logger.py + tests/unit/proxy/test_proxy_server_custom_logger.py tests/unit/proxy/test_custom_callback_input.py tests/unit/proxy/test_custom_logger_s3_gcs.py python-version: "3.12" @@ -880,7 +868,7 @@ jobs: - shard: endpoints-and-responses test-path: |- - tests/proxy_unit_tests/test_proxy_exception_mapping.py + tests/unit/proxy/test_proxy_server_exception_mapping.py tests/unit/proxy/lens tests/unit/proxy/auth/test_models_fallback_endpoint.py tests/unit/proxy/common_utils/test_check_batch_cost.py diff --git a/Makefile b/Makefile index 2a96a8a5e93..1552187e906 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ help: @echo " make check-circular-imports - Check for circular imports" @echo " make check-import-safety - Check import safety" @echo " make test - Run all tests" - @echo " make test-unit - Run unit tests (tests/unit and tests/test_litellm)" + @echo " make test-unit - Run unit tests (tests/unit)" @echo " make test-unit-llms - Run LLM provider tests (~225 files)" @echo " make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)" @echo " make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)" @@ -300,7 +300,7 @@ test: install-test-deps $(UV_RUN) pytest tests/ test-unit: install-test-deps - $(UV_RUN) pytest tests/unit tests/test_litellm -x -vv -n 4 + $(UV_RUN) pytest tests/unit -x -vv -n 4 # Matrix test targets (matching CI workflow groups) test-unit-llms: install-test-deps @@ -328,7 +328,7 @@ test-unit-other: install-test-deps $(UV_RUN) pytest tests/unit/caching tests/unit/responses tests/unit/secret_managers tests/unit/vector_stores tests/unit/a2a_protocol tests/unit/completion_extras tests/unit/containers tests/unit/enterprise tests/unit/experimental_mcp_client tests/unit/google_genai tests/unit/images tests/unit/interactions tests/unit/router_strategy tests/unit/router_utils tests/unit/types --tb=short -vv -n 4 --durations=20 test-unit-root: install-test-deps - $(UV_RUN) pytest tests/unit/test_*.py tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/unit/test_*.py --tb=short -vv -n 4 --durations=20 # Proxy unit tests (tests/unit/proxy split alphabetically) test-proxy-unit-a: install-test-deps @@ -338,7 +338,7 @@ test-proxy-unit-b: install-test-deps $(UV_RUN) pytest tests/unit/proxy/test_[p-z]*.py tests/unit/skills --tb=short -vv -n 2 --durations=20 test-integration: install-test-deps - $(UV_RUN) pytest tests/ -k "not test_litellm" + $(UV_RUN) pytest tests/ test-unit-helm: install-helm-unittest helm unittest -f 'tests/*.yaml' helm/litellm-helm diff --git a/codecov.yaml b/codecov.yaml index ae72f6d5286..5b30e78a8fb 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -71,6 +71,8 @@ flag_management: carryforward: false - name: lens-python-310 carryforward: false + - name: lens-postgres + carryforward: false - name: proxy-db-key-generation carryforward: false - name: proxy-db-auth-checks diff --git a/litellm/__init__.py b/litellm/__init__.py index 34f08da3c89..a33a78fb325 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -440,10 +440,7 @@ autorouter_presets_url: str = os.getenv( "LITELLM_AUTOROUTER_PRESETS_URL", "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/proxy/public_endpoints/autorouter_presets.json", ) -whats_new_url: str = os.getenv( - "LITELLM_WHATS_NEW_URL", - "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/proxy/public_endpoints/whats_new.json", -) +whats_new_url: str | None = os.getenv("LITELLM_WHATS_NEW_URL") suppress_debug_info: bool = False dynamodb_table_name: Optional[str] = None s3_callback_params: Optional[Dict] = None diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 715e492b159..d6c97b120c9 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2507,7 +2507,7 @@ class Logging(LiteLLMLoggingBaseClass): self.callback_duration_ms += (time.time() - _start) * 1000 return payload - def _transform_usage_objects(self, result): + def _transform_usage_objects(self, result: object) -> object: if isinstance(result, ResponsesAPIResponse): result = result.model_copy() transformed_usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(result.usage) @@ -2545,13 +2545,13 @@ class Logging(LiteLLMLoggingBaseClass): def _success_handler_helper_fn( self, - result=None, - start_time=None, - end_time=None, - cache_hit=None, + result: object = None, + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + cache_hit: bool | None = None, standard_logging_object: StandardLoggingPayload | None = None, build_logging_payload: bool = True, - ): + ) -> tuple[datetime.datetime, datetime.datetime, object]: try: if start_time is None: start_time = self.start_time diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 2070c9f29ec..2ceb0042f1f 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1,9 +1,10 @@ import base64 import time from collections.abc import Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass, replace from itertools import groupby from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Optional, TypeAlias, TypedDict, Union, cast +from typing import TYPE_CHECKING, Final, Optional, TypeAlias, TypedDict, Union, cast from pydantic import BaseModel from typing_extensions import ReadOnly, Required @@ -154,6 +155,15 @@ class _ToolCallChoice(TypedDict, total=False): _ToolCallKey: TypeAlias = tuple[int, int] +@dataclass(frozen=True, slots=True) +class _ToolCallAccumulator: + id: str | None = None + name: str | None = None + type: str | None = None + custom_name: str | None = None + provider_specific_fields: Mapping[str, object] | None = None + + class _ToolCallChunk(TypedDict): choices: ReadOnly[Sequence[_ToolCallChoice]] @@ -503,7 +513,7 @@ class ChunkProcessor: self, tool_call_chunks: Sequence["_ToolCallChunk"] ) -> list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall]: tool_calls_list: list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] = [] - tool_call_map: Final[dict[_ToolCallKey, dict[str, Any]]] = {} + tool_call_map: Final[dict[_ToolCallKey, _ToolCallAccumulator]] = {} for chunk in tool_call_chunks: choices = chunk["choices"] @@ -537,49 +547,43 @@ class ChunkProcessor: index = (choice_index, getattr(tool_call, "index", 0)) if index not in tool_call_map: - tool_call_map[index] = { - "id": None, - "name": None, - "type": None, - "custom_name": None, - "provider_specific_fields": None, - } + tool_call_map[index] = _ToolCallAccumulator() # Extract id, type, and function data (handle both dict and object) if isinstance(tool_call, dict): if fragment_id := tool_call.get("id"): - tool_call_map[index]["id"] = fragment_id + tool_call_map[index] = replace(tool_call_map[index], id=fragment_id) if fragment_type := tool_call.get("type"): - tool_call_map[index]["type"] = fragment_type + tool_call_map[index] = replace(tool_call_map[index], type=fragment_type) function = tool_call.get("function", {}) if isinstance(function, dict): if fragment_name := function.get("name"): - tool_call_map[index]["name"] = fragment_name + tool_call_map[index] = replace(tool_call_map[index], name=fragment_name) else: # function is an object if function_name := getattr(function, "name", None): - tool_call_map[index]["name"] = function_name + tool_call_map[index] = replace(tool_call_map[index], name=function_name) custom = tool_call.get("custom") if isinstance(custom, dict): if custom_name := custom.get("name"): - tool_call_map[index]["custom_name"] = custom_name + tool_call_map[index] = replace(tool_call_map[index], custom_name=custom_name) else: # tool_call is an object if hasattr(tool_call, "id") and tool_call.id: - tool_call_map[index]["id"] = tool_call.id + tool_call_map[index] = replace(tool_call_map[index], id=tool_call.id) if hasattr(tool_call, "type") and tool_call.type: - tool_call_map[index]["type"] = tool_call.type + tool_call_map[index] = replace(tool_call_map[index], type=tool_call.type) if object_function_name := getattr(getattr(tool_call, "function", None), "name", None): - tool_call_map[index]["name"] = object_function_name + tool_call_map[index] = replace(tool_call_map[index], name=object_function_name) object_custom: ChatCompletionDeltaCustomToolCallPayload | None = getattr( tool_call, "custom", None ) if object_custom is not None: if getattr(object_custom, "name", None): - tool_call_map[index]["custom_name"] = object_custom.name + tool_call_map[index] = replace(tool_call_map[index], custom_name=object_custom.name) # Preserve provider_specific_fields from streaming chunks provider_fields: object = None @@ -602,12 +606,14 @@ class ChunkProcessor: if provider_fields: # Merge provider_specific_fields if multiple chunks have them - merged_provider_fields = tool_call_map[index]["provider_specific_fields"] - if merged_provider_fields is None: - merged_provider_fields = {} - tool_call_map[index]["provider_specific_fields"] = merged_provider_fields - if isinstance(provider_fields, dict): - merged_provider_fields.update(provider_fields) + tool_call_map[index] = replace( + tool_call_map[index], + provider_specific_fields=( + {**(tool_call_map[index].provider_specific_fields or {}), **provider_fields} + if isinstance(provider_fields, dict) + else dict(tool_call_map[index].provider_specific_fields or {}) + ), + ) joined_fragments: Final = self._join_fragments_by_key_and_field( self._iter_tool_call_fragments(tool_call_chunks) @@ -616,37 +622,37 @@ class ChunkProcessor: # Convert the map to a list of tool calls for index in sorted(tool_call_map.keys()): tool_call_data = tool_call_map[index] - if tool_call_data["id"] and tool_call_data["custom_name"]: + if tool_call_data.id and tool_call_data.custom_name: tool_calls_list.append( ChatCompletionMessageCustomToolCall( - id=tool_call_data["id"], + id=tool_call_data.id, custom=ChatCompletionCustomToolCallPayload( - name=tool_call_data["custom_name"], + name=tool_call_data.custom_name, input=joined_fragments.get((index, "custom_input"), ""), ), ) ) - elif tool_call_data["id"] and tool_call_data["name"]: + elif tool_call_data.id and tool_call_data.name: combined_arguments = joined_fragments.get((index, "arguments"), "") or "{}" # Build function - provider_specific_fields should be on tool_call level, not function level function = Function( arguments=combined_arguments, - name=tool_call_data["name"], + name=tool_call_data.name, ) # Prepare params for ChatCompletionMessageToolCall - tool_call_params = { - "id": tool_call_data["id"], - "function": function, - "type": tool_call_data["type"] or "function", - } - # Add provider_specific_fields if present (for thought signatures in Gemini 3) - if tool_call_data.get("provider_specific_fields"): - tool_call_params["provider_specific_fields"] = tool_call_data["provider_specific_fields"] - - tool_call = ChatCompletionMessageToolCall(**tool_call_params) + tool_call = ChatCompletionMessageToolCall( + id=tool_call_data.id, + function=function, + type=tool_call_data.type or "function", + **( + {"provider_specific_fields": dict(tool_call_data.provider_specific_fields)} + if tool_call_data.provider_specific_fields + else {} + ), + ) tool_calls_list.append(tool_call) return tool_calls_list diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index 207ed249866..0813ef58528 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -474,7 +474,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): Returns: AgentCoreParsedResponse: Parsed response data """ - content_type: Final = raw_response.headers.get("content-type", "").lower() + content_type: Final = ", ".join(httpx.Headers(raw_response.headers).get_list("content-type")).lower() verbose_logger.debug("AgentCore response Content-Type: %s", content_type) # Parse response based on content type @@ -698,7 +698,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): ) # Check if response is JSON (agent used sync return) instead of SSE - content_type: Final = response.headers.get("content-type", "").lower() + content_type: Final = ", ".join(httpx.Headers(response.headers).get_list("content-type")).lower() if "application/json" in content_type: verbose_logger.debug( "AgentCore streaming: received JSON response instead of SSE, converting to single-chunk stream" @@ -920,7 +920,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): ) # Check if response is JSON (agent used sync return) instead of SSE - content_type: Final = response.headers.get("content-type", "").lower() + content_type: Final = ", ".join(httpx.Headers(response.headers).get_list("content-type")).lower() if "application/json" in content_type: verbose_logger.debug( "AgentCore streaming: received JSON response instead of SSE, converting to single-chunk stream" diff --git a/litellm/llms/gemini/count_tokens/handler.py b/litellm/llms/gemini/count_tokens/handler.py index 67e84f6da24..ca47ac67957 100644 --- a/litellm/llms/gemini/count_tokens/handler.py +++ b/litellm/llms/gemini/count_tokens/handler.py @@ -1,6 +1,9 @@ +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import ConfigDict, RootModel, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm.llms.custom_httpx.http_handler import get_async_httpx_client @@ -12,6 +15,16 @@ else: GenerateContentContentListUnionDict = Any +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _GeminiCountTokensResponse(TypedDict): + totalTokens: NotRequired[ReadOnly[int | float]] + tokenizer_used: NotRequired[ReadOnly[str]] + + +class _GeminiCountTokensResponseRoot(RootModel[_GeminiCountTokensResponse]): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + class GoogleAIStudioTokenCounter: def _clean_contents_for_gemini_api(self, contents: Any) -> object: """ @@ -79,7 +92,7 @@ class GoogleAIStudioTokenCounter: async def acount_tokens( self, - contents: Any, + contents: Sequence[Mapping[str, object]] | None, model: str, api_key: str | None = None, api_base: str | None = None, @@ -143,8 +156,8 @@ class GoogleAIStudioTokenCounter: response.raise_for_status() # Parse response - result: Final = response.json() - return result + result: Final = _GeminiCountTokensResponseRoot.model_validate(response.json()).root + return dict(result) except httpx.HTTPStatusError as e: error_msg = f"Google Gen AI Studio API error: {e.response.status_code} - {e.response.text}" diff --git a/litellm/llms/openrouter/image_edit/transformation.py b/litellm/llms/openrouter/image_edit/transformation.py index 96b16e3e510..a73a2fe4192 100644 --- a/litellm/llms/openrouter/image_edit/transformation.py +++ b/litellm/llms/openrouter/image_edit/transformation.py @@ -41,11 +41,14 @@ Response format: """ import base64 +from collections.abc import Mapping, Sequence from io import BufferedReader, BytesIO from typing import TYPE_CHECKING, Any, Final, cast import httpx from httpx._types import RequestFiles +from pydantic import ConfigDict, RootModel, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm.images.utils import ImageEditRequestUtils @@ -71,6 +74,52 @@ else: LiteLLMLoggingObj = Any +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _ImageUrl(TypedDict): + url: ReadOnly[NotRequired[str | None]] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _Image(TypedDict): + image_url: ReadOnly[NotRequired[_ImageUrl]] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _Message(TypedDict): + images: ReadOnly[NotRequired[Sequence[_Image]]] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _Choice(TypedDict): + message: ReadOnly[NotRequired[_Message]] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _TokensDetails(TypedDict): + image_tokens: ReadOnly[NotRequired[int | float]] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _Usage(TypedDict): + prompt_tokens: ReadOnly[NotRequired[int | float]] + total_tokens: ReadOnly[NotRequired[int | float]] + completion_tokens_details: ReadOnly[NotRequired[_TokensDetails]] + prompt_tokens_details: ReadOnly[NotRequired[_TokensDetails | None]] + cost: ReadOnly[NotRequired[int | float | str | None]] + cost_details: ReadOnly[NotRequired[Mapping[str, object] | None]] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _OpenRouterImageEditResponse(TypedDict): + choices: ReadOnly[NotRequired[Sequence[_Choice]]] + usage: ReadOnly[NotRequired[_Usage | None]] + model: ReadOnly[NotRequired[object]] + + +class _OpenRouterImageEditResponseRoot(RootModel[_OpenRouterImageEditResponse]): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + class OpenRouterImageEditConfig(BaseImageEditConfig): """ Configuration for OpenRouter image editing via chat completions. @@ -201,7 +250,7 @@ class OpenRouterImageEditConfig(BaseImageEditConfig): logging_obj: LiteLLMLoggingObj, ) -> ImageResponse: try: - response_json: Final = raw_response.json() + response_json: Final[object] = raw_response.json() except Exception as e: raise OpenRouterException( message=f"Error parsing OpenRouter response: {e}", @@ -213,7 +262,8 @@ class OpenRouterImageEditConfig(BaseImageEditConfig): model_response.data = [] try: - choices: Final = response_json.get("choices", []) + payload: Final = _OpenRouterImageEditResponseRoot.model_validate(response_json).root + choices: Final = payload.get("choices", []) for choice in choices: message = choice.get("message", {}) @@ -252,7 +302,7 @@ class OpenRouterImageEditConfig(BaseImageEditConfig): headers={}, ) - self._set_usage_and_cost(model_response, response_json, model) + self._set_usage_and_cost(model_response, payload, model) return model_response def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: @@ -303,11 +353,11 @@ class OpenRouterImageEditConfig(BaseImageEditConfig): def _set_usage_and_cost( self, model_response: ImageResponse, - response_json: dict, + payload: _OpenRouterImageEditResponse, model: str, ) -> None: """Extract and set usage and cost information from OpenRouter response.""" - usage_data: Final = response_json.get("usage", {}) + usage_data: Final = payload.get("usage", {}) if usage_data: prompt_tokens: Final = usage_data.get("prompt_tokens", 0) total_tokens: Final = usage_data.get("total_tokens", 0) @@ -347,7 +397,7 @@ class OpenRouterImageEditConfig(BaseImageEditConfig): ) response_cost_details.update(cost_details) - model_response.hidden_params["model"] = response_json.get("model", model) + model_response.hidden_params["model"] = payload.get("model", model) def _read_image_bytes(self, image: FileTypes) -> bytes: """Read raw bytes from various image input types.""" diff --git a/litellm/llms/ragflow/vector_stores/transformation.py b/litellm/llms/ragflow/vector_stores/transformation.py index 38a06a37f7e..007b047cc1b 100644 --- a/litellm/llms/ragflow/vector_stores/transformation.py +++ b/litellm/llms/ragflow/vector_stores/transformation.py @@ -2,6 +2,8 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import ConfigDict, RootModel, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.secret_managers.main import get_secret_str @@ -22,6 +24,25 @@ else: LiteLLMLoggingObj = Any +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _RAGFlowDataset(TypedDict): + id: ReadOnly[NotRequired[str | None]] + name: ReadOnly[NotRequired[str | None]] + create_time: ReadOnly[NotRequired[int | float | None]] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _RAGFlowCreateResponse(TypedDict): + data: ReadOnly[NotRequired[_RAGFlowDataset]] + + +class _RAGFlowCreateResponseRoot(RootModel[_RAGFlowCreateResponse]): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + +_RAGFLOW_EMPTY_DATA: Final[_RAGFlowDataset] = {} + + class RAGFlowVectorStoreConfig(BaseVectorStoreConfig): """Vector store configuration for RAGFlow datasets.""" @@ -185,7 +206,8 @@ class RAGFlowVectorStoreConfig(BaseVectorStoreConfig): headers=response.headers, ) - data: Final = response_json.get("data", {}) + create_response: Final = _RAGFlowCreateResponseRoot.model_validate(response_json).root + data: Final = create_response.get("data", _RAGFLOW_EMPTY_DATA) # Extract dataset ID dataset_id: Final = data.get("id") diff --git a/litellm/llms/sagemaker/common_utils.py b/litellm/llms/sagemaker/common_utils.py index 1f7fba3c4c4..7772ca6869a 100644 --- a/litellm/llms/sagemaker/common_utils.py +++ b/litellm/llms/sagemaker/common_utils.py @@ -4,7 +4,8 @@ from collections.abc import AsyncIterator, Iterator, Mapping from typing import Final import httpx -from pydantic import ConfigDict, TypeAdapter +from pydantic import ConfigDict, RootModel, TypeAdapter, with_config +from typing_extensions import ReadOnly, Required, TypedDict import litellm from litellm import verbose_logger @@ -15,6 +16,38 @@ from litellm.types.utils import StreamingChatCompletionChunk _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _SageMakerToken(TypedDict, total=False): + text: ReadOnly[str | None] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _SageMakerChunkData(TypedDict, total=False): + token: ReadOnly[_SageMakerToken | None] + index: ReadOnly[int | None] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _SageMakerMessagesChunkData(TypedDict, total=False): + choices: ReadOnly[Required[list[Mapping[str, object]]]] + + +class _SageMakerChunkDataRoot(RootModel[_SageMakerChunkData]): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + +class _SageMakerMessagesChunkDataRoot(RootModel[_SageMakerMessagesChunkData]): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + +def _parse_sagemaker_chunk_data(data: str) -> _SageMakerChunkData: + return _SageMakerChunkDataRoot.model_validate(json.loads(data)).root + + +def _parse_sagemaker_messages_chunk_data(data: str) -> _SageMakerMessagesChunkData: + return _SageMakerMessagesChunkDataRoot.model_validate(json.loads(data)).root + + def _load_sagemaker_response_stream_shape(): try: from botocore.loaders import Loader @@ -62,12 +95,12 @@ class AWSEventStreamDecoder: self.content_blocks: list = [] self.is_messages_api = is_messages_api - def _chunk_parser_messages_api(self, chunk_data: dict) -> StreamingChatCompletionChunk: + def _chunk_parser_messages_api(self, chunk_data: _SageMakerMessagesChunkData) -> StreamingChatCompletionChunk: openai_chunk: Final = StreamingChatCompletionChunk(**{"model": self.model, **chunk_data}) return openai_chunk - def _chunk_parser(self, chunk_data: dict) -> GChunk: + def _chunk_parser(self, chunk_data: _SageMakerChunkData) -> GChunk: verbose_logger.debug("in sagemaker chunk parser, chunk_data %s", chunk_data) _token: Final = chunk_data.get("token", {}) or {} _index: Final = chunk_data.get("index", None) or 0 @@ -113,11 +146,12 @@ class AWSEventStreamDecoder: # Try to parse the accumulated JSON try: - _data = json.loads(accumulated_json) if self.is_messages_api: - yield self._chunk_parser_messages_api(chunk_data=_data) + yield self._chunk_parser_messages_api( + chunk_data=_parse_sagemaker_messages_chunk_data(accumulated_json) + ) else: - yield self._chunk_parser(chunk_data=_data) + yield self._chunk_parser(chunk_data=_parse_sagemaker_chunk_data(accumulated_json)) # Reset accumulated_json after successful parsing accumulated_json = "" except json.JSONDecodeError: @@ -127,11 +161,12 @@ class AWSEventStreamDecoder: # Handle any remaining data after the iterator is exhausted if accumulated_json: try: - _data = json.loads(accumulated_json) if self.is_messages_api: - yield self._chunk_parser_messages_api(chunk_data=_data) + yield self._chunk_parser_messages_api( + chunk_data=_parse_sagemaker_messages_chunk_data(accumulated_json) + ) else: - yield self._chunk_parser(chunk_data=_data) + yield self._chunk_parser(chunk_data=_parse_sagemaker_chunk_data(accumulated_json)) except json.JSONDecodeError: # Handle or log any unparseable data at the end verbose_logger.error("Warning: Unparseable JSON data remained: %s", accumulated_json) @@ -161,11 +196,12 @@ class AWSEventStreamDecoder: accumulated_json += message # Try to parse the accumulated JSON - _data = json.loads(accumulated_json) if self.is_messages_api: - yield self._chunk_parser_messages_api(chunk_data=_data) + yield self._chunk_parser_messages_api( + chunk_data=_parse_sagemaker_messages_chunk_data(accumulated_json) + ) else: - yield self._chunk_parser(chunk_data=_data) + yield self._chunk_parser(chunk_data=_parse_sagemaker_chunk_data(accumulated_json)) # Reset accumulated_json after successful parsing accumulated_json = "" except json.JSONDecodeError: @@ -181,11 +217,12 @@ class AWSEventStreamDecoder: # Handle any remaining data after the iterator is exhausted if accumulated_json: try: - _data = json.loads(accumulated_json) if self.is_messages_api: - yield self._chunk_parser_messages_api(chunk_data=_data) + yield self._chunk_parser_messages_api( + chunk_data=_parse_sagemaker_messages_chunk_data(accumulated_json) + ) else: - yield self._chunk_parser(chunk_data=_data) + yield self._chunk_parser(chunk_data=_parse_sagemaker_chunk_data(accumulated_json)) except json.JSONDecodeError: # Handle or log any unparseable data at the end verbose_logger.error("Warning: Unparseable JSON data remained: %s", accumulated_json) diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 2d35dd9b480..ebbe2c59886 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -1,6 +1,9 @@ +from collections.abc import Sequence from typing import Final, Literal import httpx +from pydantic import ConfigDict, RootModel, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -13,10 +16,6 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, ) from litellm.llms.openai.openai import AllMessageValues -from litellm.types.llms.vertex_ai import ( - CachedContentListAllResponseBody, - VertexAICachedContentResponseObject, -) from litellm.utils import is_prompt_caching_valid_prompt from ..common_utils import VertexAIError, get_vertex_base_url @@ -32,6 +31,31 @@ local_cache_obj: Final = Cache(type=LiteLLMCacheType.LOCAL) # only used for cal MAX_PAGINATION_PAGES: Final = 100 # Reasonable upper bound for pagination +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _CachedContentItem(TypedDict): + displayName: ReadOnly[NotRequired[object]] + name: ReadOnly[NotRequired[str | None]] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _CachedContentListBody(TypedDict): + cachedContents: ReadOnly[NotRequired[Sequence[_CachedContentItem]]] + nextPageToken: ReadOnly[NotRequired[str | None]] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _CreatedCachedContent(TypedDict): + name: ReadOnly[NotRequired[str | None]] + + +class _CachedContentListBodyRoot(RootModel[_CachedContentListBody]): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + +class _CreatedCachedContentRoot(RootModel[_CreatedCachedContent]): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + class ContextCachingEndpoints(VertexBase): """ Covers context caching endpoints for Vertex AI + Google AI Studio @@ -157,25 +181,20 @@ class ContextCachingEndpoints(VertexBase): except Exception as e: raise VertexAIError(status_code=500, message=str(e)) - raw_response = resp.json() + raw_response = _CachedContentListBodyRoot.model_validate(resp.json()).root logging_obj.post_call(original_response=raw_response) if "cachedContents" not in raw_response: return None - all_cached_items = CachedContentListAllResponseBody(**raw_response) - - if "cachedContents" not in all_cached_items: - return None - # Check current page for matching cache_key - for cached_item in all_cached_items["cachedContents"]: + for cached_item in raw_response["cachedContents"]: display_name = cached_item.get("displayName") if display_name is not None and display_name == cache_key: return cached_item.get("name") # Check if there are more pages - page_token = all_cached_items.get("nextPageToken") + page_token = raw_response.get("nextPageToken") if not page_token: # No more pages, cache not found break @@ -249,25 +268,20 @@ class ContextCachingEndpoints(VertexBase): except Exception as e: raise VertexAIError(status_code=500, message=str(e)) - raw_response = resp.json() + raw_response = _CachedContentListBodyRoot.model_validate(resp.json()).root logging_obj.post_call(original_response=raw_response) if "cachedContents" not in raw_response: return None - all_cached_items = CachedContentListAllResponseBody(**raw_response) - - if "cachedContents" not in all_cached_items: - return None - # Check current page for matching cache_key - for cached_item in all_cached_items["cachedContents"]: + for cached_item in raw_response["cachedContents"]: display_name = cached_item.get("displayName") if display_name is not None and display_name == cache_key: return cached_item.get("name") # Check if there are more pages - page_token = all_cached_items.get("nextPageToken") + page_token = raw_response.get("nextPageToken") if not page_token: # No more pages, cache not found break @@ -424,14 +438,11 @@ class ContextCachingEndpoints(VertexBase): except httpx.TimeoutException: raise VertexAIError(status_code=408, message="Timeout error occurred.") - raw_response_cached: Final = response.json() - cached_content_response_obj: Final = VertexAICachedContentResponseObject( - name=raw_response_cached.get("name"), model=raw_response_cached.get("model") - ) + raw_response_cached: Final = _CreatedCachedContentRoot.model_validate(response.json()).root return ( non_cached_messages, optional_params, - cached_content_response_obj["name"], + raw_response_cached.get("name"), ) async def async_check_and_create_cache( @@ -580,14 +591,11 @@ class ContextCachingEndpoints(VertexBase): except httpx.TimeoutException: raise VertexAIError(status_code=408, message="Timeout error occurred.") - raw_response_cached: Final = response.json() - cached_content_response_obj: Final = VertexAICachedContentResponseObject( - name=raw_response_cached.get("name"), model=raw_response_cached.get("model") - ) + raw_response_cached: Final = _CreatedCachedContentRoot.model_validate(response.json()).root return ( non_cached_messages, optional_params, - cached_content_response_obj["name"], + raw_response_cached.get("name"), ) def get_cache(self): diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index fb846d6b196..93e0a183a2b 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -2209,10 +2209,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): @staticmethod def _set_stream_metadata_on_response( model_response: Any, - grounding_metadata: list[dict], - url_context_metadata: list[dict], - safety_ratings: list[dict], - citation_metadata: list[dict], + grounding_metadata: Sequence[Mapping[str, object]], + url_context_metadata: Sequence[Mapping[str, object]], + safety_ratings: Sequence[Mapping[str, object]], + citation_metadata: Sequence[Mapping[str, object]], ) -> None: setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata) if grounding_metadata: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 782ef2f2ff2..780adab42ab 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -63565,7 +63565,7 @@ }, "azure_ai/kimi-k2.7-code": { "cache_read_input_token_cost": 1.9e-07, - "deprecation_date": "2026-10-03", + "deprecation_date": "2027-04-16", "input_cost_per_token": 9.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index 1d4a5d48a65..21c94b76d21 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -5,6 +5,8 @@ from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal import httpx from fastapi import HTTPException +from pydantic import ConfigDict, RootModel, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -65,6 +67,55 @@ from litellm.types.utils import ( GUARDRAIL_NAME: Final = "model_armor" + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _MatchResult(TypedDict): + matchState: NotRequired[ReadOnly[str]] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _DeidentifyData(TypedDict): + text: NotRequired[ReadOnly[str]] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _DeidentifyResult(TypedDict): + matchState: NotRequired[ReadOnly[str]] + data: NotRequired[ReadOnly[_DeidentifyData]] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _SdpResult(TypedDict): + inspectResult: NotRequired[ReadOnly[_MatchResult]] + deidentifyResult: NotRequired[ReadOnly[_DeidentifyResult]] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _Filter(TypedDict): + raiFilterResult: NotRequired[ReadOnly[_MatchResult]] + piAndJailbreakFilterResult: NotRequired[ReadOnly[_MatchResult]] + maliciousUriFilterResult: NotRequired[ReadOnly[_MatchResult]] + csamFilterFilterResult: NotRequired[ReadOnly[_MatchResult]] + virusScanFilterResult: NotRequired[ReadOnly[_MatchResult]] + sdpFilterResult: NotRequired[ReadOnly[_SdpResult | None]] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _SanitizationResult(TypedDict): + filterResults: NotRequired[ReadOnly[Mapping[str, _Filter] | Sequence[_Filter] | str | int | float | bool | None]] + + +@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) +class _ModelArmorResponse(TypedDict): + sanitizationResult: NotRequired[ReadOnly[_SanitizationResult]] + sanitizedText: NotRequired[ReadOnly[str | None]] + text: NotRequired[ReadOnly[str | None]] + + +class _ModelArmorResponseRoot(RootModel[_ModelArmorResponse]): + model_config = ConfigDict(frozen=True, hide_input_in_errors=True) + + # Only these carry the finished output; response.created carries an empty body _RESPONSES_TERMINAL_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete", "response.failed"}) @@ -99,10 +150,11 @@ class ModelArmorAPIError(Exception): _SCANNED_CONTENT_KEYS: Final = frozenset({"text", "sanitizedText", "findings", "maliciousUriMatchedItems"}) + RedactablePayload = dict | list | str | int | float | bool | None -def _redact_scanned_content(payload: RedactablePayload, depth: int = 0) -> RedactablePayload: +def _redact_scanned_content(payload: object, depth: int = 0) -> object: if depth >= DEFAULT_MAX_RECURSE_DEPTH: return "[REDACTED]" if isinstance(payload, dict): @@ -223,12 +275,12 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): return f"Model Armor API error (upstream {status_code})" return f"Model Armor API error (upstream {status_code}): {response_text}" - def _build_block_error_detail(self, message: str, armor_response: RedactablePayload) -> dict: + def _build_block_error_detail(self, message: str, armor_response: Mapping[str, object]) -> dict: if self.sanitize_error_detail: return {"error": message} return {"error": message, "model_armor_response": armor_response} - def _build_logging_response(self, armor_response: RedactablePayload) -> RedactablePayload: + def _build_logging_response(self, armor_response: Mapping[str, object]) -> object: if self.sanitize_error_detail: return _redact_scanned_content(armor_response) return armor_response @@ -286,7 +338,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): request_data: dict | None = None, file_bytes: bytes | None = None, file_type: str | None = None, - ) -> dict: + ) -> _ModelArmorResponse: """ Make request to Model Armor API. Supports both text and file prompt sanitization. If file_bytes and file_type are provided, file prompt sanitization is performed. @@ -361,8 +413,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): json_response: Final = response.json() if hasattr(json_response, "__await__"): - return await json_response - return json_response + return _ModelArmorResponseRoot.model_validate(await json_response).root + return _ModelArmorResponseRoot.model_validate(json_response).root def sanitize_file_prompt(self, file_bytes: bytes, file_type: str, source: str = "user_prompt") -> dict: """ @@ -378,7 +430,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): else: return {"modelResponseData": {"byteItem": {"byteDataType": file_type, "byteData": base64_data}}} - def _should_block_content(self, armor_response: Mapping[str, object], allow_sanitization: bool = False) -> bool: + def _should_block_content(self, armor_response: _ModelArmorResponse, allow_sanitization: bool = False) -> bool: """Check if Model Armor response indicates content should be blocked, including both inspectResult and deidentifyResult.""" for filt in self._filter_result_items(armor_response): # Check RAI, PI/Jailbreak, Malicious URI, CSAM, Virus scan as before @@ -404,7 +456,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): # Fallback dict code removed; all cases handled above return False - def _get_sanitized_content(self, armor_response: Mapping[str, Any]) -> str | None: + def _get_sanitized_content(self, armor_response: _ModelArmorResponse) -> str | None: """ Get the sanitized content from a Model Armor response, if available. Looks for sanitized text in deidentifyResult, and falls back to root-level fields if not found. @@ -434,7 +486,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): return armor_response.get("sanitizedText") or armor_response.get("text") @staticmethod - def _filter_result_items(armor_response: Mapping[str, Any]) -> Sequence[Any]: + def _filter_result_items(armor_response: _ModelArmorResponse) -> Sequence[_Filter]: """Every filter result in a scan response. filterResults is a dict of named filters on most templates and a list on some, so both @@ -447,7 +499,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): return filter_results return [] - def _has_deidentify_match(self, armor_response: Mapping[str, object]) -> bool: + def _has_deidentify_match(self, armor_response: _ModelArmorResponse) -> bool: """Whether an SDP de-identify filter matched, i.e. Model Armor owes this response a redaction.""" for filter_entry in self._filter_result_items(armor_response): sdp = filter_entry.get("sdpFilterResult") @@ -457,7 +509,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): def _resolve_streaming_outcome( self, - armor_response: Mapping[str, object], + armor_response: _ModelArmorResponse, assembled_response: object, content: str, ) -> tuple[bool, str | None]: @@ -489,7 +541,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): return False, sanitized_content @staticmethod - def _append_armor_response(existing: object, armor_response: Mapping[str, object]) -> object: + def _append_armor_response(existing: object, armor_response: object) -> object: """Accumulate scan responses so a later text scan does not drop an earlier file scan. Returns the single response on its own (backward compatible) and a list once a request diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 5a313b14c4a..4e46a070ec2 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -245,7 +245,7 @@ class _TransactionFactory(Protocol): class _ModelTransactionClient(LiteLLMBaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True, from_attributes=True) + model_config = ConfigDict(arbitrary_types_allowed=True, from_attributes=True, hide_input_in_errors=True) tx: _TransactionFactory @@ -1856,8 +1856,8 @@ async def delete_team_models( Returns the model_ids that were deleted. """ deleted_model_ids: Final[list[str]] = [] - async with prisma_client.db.tx() as tx_ctx: - tx: Final[_TxModelTables] = tx_ctx + transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db) + async with transaction_client.tx() as tx: for team_id in team_ids: rows = await _get_team_deployments(team_id, prisma_client, table=tx.litellm_proxymodeltable) model_ids = [row.model_id for row in rows] diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index ed5843f4bce..a4d2977a971 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -41,7 +41,7 @@ if TYPE_CHECKING: import jwt from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response, status from fastapi.responses import RedirectResponse -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError import litellm from litellm._internal_context import with_service_target @@ -237,6 +237,12 @@ class CliSsoTeamDetail(LiteLLMBaseModel): _CLI_SSO_TEAM_DETAILS_ADAPTER: Final = TypeAdapter(tuple[CliSsoTeamDetail, ...]) +_CLI_SSO_FLOW_JSON_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter( + dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True) +) +_SSO_USERINFO_RESPONSE_ADAPTER: Final[TypeAdapter[dict[str, object] | None]] = TypeAdapter( + dict[str, object] | None, config=ConfigDict(strict=True, hide_input_in_errors=True) +) _TEAMLESS_CLI_SSO_TEAM_DETAIL: Final = CliSsoTeamDetail(team_models=()) @@ -371,7 +377,7 @@ def _get_cli_sso_flow_or_raise(login_id: str | None, cache: DualCache) -> dict: flow = _read_cli_sso_flow(cache, _get_cli_sso_flow_cache_key(cast(str, login_id))) if isinstance(flow, str): try: - flow = _as_object(json.loads(flow)) + flow = _CLI_SSO_FLOW_JSON_ADAPTER.validate_python(json.loads(flow)) except ValueError: flow = None if not isinstance(flow, dict) or "poll_secret_hash" not in flow: @@ -4199,7 +4205,7 @@ class SSOAuthenticationHandler: ) if resp.status_code == 200: try: - userinfo_raw: Final[dict[str, object] | None] = resp.json() + userinfo_raw: Final = _SSO_USERINFO_RESPONSE_ADAPTER.validate_python(resp.json()) if not userinfo_raw: # JSON null (None) or empty dict ({}) — no identity claims. # Treat as failure so id_token fallback can be attempted. diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 75e8a5af6a5..26bef3b3a61 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -591,8 +591,8 @@ async def _fetch_remote_whats_new(url: str) -> WhatsNewResponse: return WhatsNewResponse.model_validate_json(await _fetch_remote_bytes(url)) -async def _resolve_whats_new(url: str, fetch: Callable[[str], Awaitable[WhatsNewResponse]]) -> WhatsNewResponse: - if os.getenv("LITELLM_LOCAL_WHATS_NEW", "").lower() == "true": +async def _resolve_whats_new(url: str | None, fetch: Callable[[str], Awaitable[WhatsNewResponse]]) -> WhatsNewResponse: + if url is None or os.getenv("LITELLM_LOCAL_WHATS_NEW", "").lower() == "true": return _load_bundled_whats_new() try: return await fetch(url) @@ -611,7 +611,7 @@ class _WhatsNewCache: async def get_whats_new( - url: str, + url: str | None, fetch: Callable[[str], Awaitable[WhatsNewResponse]] = _fetch_remote_whats_new, ) -> WhatsNewResponse: cached: Final = _WhatsNewCache.launches @@ -637,10 +637,10 @@ async def get_public_whats_new() -> WhatsNewResponse: """ Return the launches the dashboard Home page shows under What's new. - Resolved once per process: fetched from ``litellm.whats_new_url`` (override with ``LITELLM_WHATS_NEW_URL``) - on the first request, falling back to the list bundled with the package on any failure, so an airgapped - proxy serves the bundled list without retrying. Set ``LITELLM_LOCAL_WHATS_NEW=True`` to skip the fetch. - A restart picks up a newly published list. + Serves the list bundled with this release. Set ``LITELLM_WHATS_NEW_URL`` to fetch a remote list instead, + resolved once per process on the first request and falling back to the bundled list on any failure, so an + airgapped proxy serves the bundled list without retrying. ``LITELLM_LOCAL_WHATS_NEW=True`` forces the + bundled list even when a URL is set. A restart picks up a newly published list. """ return await get_whats_new(url=litellm.whats_new_url) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 782ef2f2ff2..780adab42ab 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -63565,7 +63565,7 @@ }, "azure_ai/kimi-k2.7-code": { "cache_read_input_token_cost": 1.9e-07, - "deprecation_date": "2026-10-03", + "deprecation_date": "2027-04-16", "input_cost_per_token": 9.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, diff --git a/tests/agent_tests/local_only_agent_tests/local_vertex_agent.py b/tests/agent_tests/local_only_agent_tests/local_vertex_agent.py deleted file mode 100644 index d31db00d234..00000000000 --- a/tests/agent_tests/local_only_agent_tests/local_vertex_agent.py +++ /dev/null @@ -1,157 +0,0 @@ -""" -Test script for Vertex AI Reasoning Engine. - -This script demonstrates how to: -1. Authenticate with Google Cloud -2. Send queries to a Vertex AI Reasoning Engine using the :query endpoint - -Usage: - python local_vertex_agent.py - -Requirements: - pip install httpx google-auth -""" - -import asyncio -import json -from uuid import uuid4 - -from google.auth import default -from google.auth.transport.requests import Request -import httpx - -# Configuration - update these for your agent -PROJECT_ID = "test-gcp-project-id-123" # Your GCP project ID (test value) -LOCATION = "us-central1" # Your agent's location - -# For Reasoning Engines, use just the numeric ID at the end -REASONING_ENGINE_ID = "8263861224643493888" - -# The project number from the resource name -PROJECT_NUMBER = "1060139831167" - - -async def main(): - """Main function to test Vertex AI Reasoning Engine.""" - - # Step 1: Authenticate with Google Cloud - print("Step 1: Authenticating with Google Cloud...") - credentials, project = default( - scopes=["https://www.googleapis.com/auth/cloud-platform"] - ) - credentials.refresh(Request()) - print(f"Authenticated! Project: {project}") - print(f"Token (first 20 chars): {credentials.token[:20]}...") - - # Step 2: Build the endpoint URL - base_url = f"https://{LOCATION}-aiplatform.googleapis.com" - resource_path = f"projects/{PROJECT_NUMBER}/locations/{LOCATION}/reasoningEngines/{REASONING_ENGINE_ID}" - - # The Reasoning Engine uses :query endpoint with specific format - query_url = f"{base_url}/v1beta1/{resource_path}:query" - stream_url = f"{base_url}/v1beta1/{resource_path}:streamQuery" - - print(f"\nQuery URL: {query_url}") - print(f"Stream URL: {stream_url}") - - # Step 3: Create authenticated httpx client - print("\nStep 2: Creating authenticated HTTP client...") - client = httpx.AsyncClient( - headers={ - "Authorization": f"Bearer {credentials.token}", - "Content-Type": "application/json", - }, - timeout=120.0, - ) - - # Step 4: Build the query request (non-streaming) - # Note: For non-streaming, we need to: - # 1. Create a session - # 2. Use the streaming endpoint with stream_query method - # The :query endpoint only supports session management methods - - user_id = f"test-user-{uuid4().hex[:8]}" - - # First create a session - create_session_request = { - "class_method": "async_create_session", - "input": { - "user_id": user_id, - }, - } - - print(f"\nStep 3: Creating session...") - print(f"User ID: {user_id}") - - async with client: - # Create session - print(f"\nSending to: {query_url}") - response = await client.post(query_url, json=create_session_request) - print(f"Create session status: {response.status_code}") - - if response.status_code == 200: - session_data = response.json() - print(f"Session created:\n{json.dumps(session_data, indent=2)}") - - # Extract session_id from response - session_id = session_data.get("output", {}).get("id") or session_data.get( - "output", {} - ).get("session_id") - print(f"\nSession ID: {session_id}") - - # Now send the actual query via streamQuery - query_request = { - "class_method": "stream_query", - "input": { - "message": "Hello! What can you do?", - "user_id": user_id, - "session_id": session_id, - }, - } - - print(f"\nStep 4: Sending query via streamQuery...") - print(f"Request:\n{json.dumps(query_request, indent=2)}") - - # Use streaming endpoint but collect full response - async with client.stream( - "POST", stream_url, json=query_request - ) as stream_response: - print(f"Query status: {stream_response.status_code}") - - if stream_response.status_code == 200: - print("\nResponse:") - full_response = "" - async for line in stream_response.aiter_lines(): - if line: - full_response = line # Keep last line (full response) - - # Parse and display - try: - data = json.loads(full_response) - # Extract the text from the response - content = data.get("content", {}) - parts = content.get("parts", []) - for part in parts: - if "text" in part: - print(f"\nAgent response:\n{part['text']}") - except: - print(full_response) - else: - content = await stream_response.aread() - print(f"Error: {content.decode()}") - else: - print(f"Error creating session: {response.text}") - - -if __name__ == "__main__": - print("=" * 60) - print("Vertex AI Reasoning Engine Test Script") - print("=" * 60) - print(f"\nConfiguration:") - print(f" PROJECT_ID: {PROJECT_ID}") - print(f" PROJECT_NUMBER: {PROJECT_NUMBER}") - print(f" LOCATION: {LOCATION}") - print(f" REASONING_ENGINE_ID: {REASONING_ENGINE_ID}") - print() - - asyncio.run(main()) diff --git a/tests/agent_tests/local_only_agent_tests/test_a2a.py b/tests/agent_tests/local_only_agent_tests/test_a2a.py deleted file mode 100644 index eb0c4ddfc50..00000000000 --- a/tests/agent_tests/local_only_agent_tests/test_a2a.py +++ /dev/null @@ -1,353 +0,0 @@ -""" -Test for LiteLLM A2A module. - -Run with: - pytest tests/agent_tests/test_a2a.py -v -s -""" - -import asyncio -import json -from typing import Optional -from uuid import uuid4 - -import pytest - -import litellm -from litellm.integrations.custom_logger import CustomLogger -from litellm.types.utils import StandardLoggingPayload - -from a2a.types import MessageSendParams, SendMessageRequest - - -@pytest.mark.asyncio -async def test_asend_message_with_client_decorator(): - """ - Test asend_message standalone function with @client decorator. - This tests the LiteLLM logging integration. - """ - litellm.turn_on_debug() - from litellm.a2a_protocol import asend_message, create_a2a_client - - # Create the A2A client first - a2a_client = await create_a2a_client(base_url="http://localhost:10001") - - # Build the request matching A2A SDK spec - send_message_payload = { - "message": { - "role": "user", - "parts": [ - { - "kind": "text", - "text": "Hello from @client decorated function!", - } - ], - "messageId": uuid4().hex, - }, - } - - request = SendMessageRequest( - id=str(uuid4()), - params=MessageSendParams(**send_message_payload), - ) - - # Send message using standalone function with @client decorator - response = await asend_message(a2a_client=a2a_client, request=request) - - # Print response for debugging - print("\n=== A2A Response (standalone with @client) ===") - print(response.model_dump(mode="json", exclude_none=True)) - - # Basic assertions - assert response is not None - - -class TestA2ALogger(CustomLogger): - """Custom logger to capture A2A logging payloads for testing.""" - - def __init__(self): - self.standard_logging_payload: Optional[StandardLoggingPayload] = None - self.logged_kwargs: Optional[dict] = None - self.log_success_called = False - super().__init__() - - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - print("TestA2ALogger: async_log_success_event called") - self.log_success_called = True - self.logged_kwargs = kwargs - self.standard_logging_payload = kwargs.get("standard_logging_object", None) - print(f"Captured standard_logging_payload: {self.standard_logging_payload}") - - -@pytest.mark.asyncio -async def test_a2a_logging_payload(): - """ - Test that A2A calls create a standard logging payload. - Validates the @client decorator integration with LiteLLM logging. - """ - # Reset callbacks and set up custom logger - litellm.logging_callback_manager._reset_all_callbacks() - test_logger = TestA2ALogger() - litellm.callbacks = [test_logger] - - from litellm.a2a_protocol import asend_message, create_a2a_client - - # Create the A2A client first - a2a_client = await create_a2a_client(base_url="http://localhost:10001") - - # Build the request - send_message_payload = { - "message": { - "role": "user", - "parts": [ - { - "kind": "text", - "text": "Hello! Testing logging payload.", - } - ], - "messageId": uuid4().hex, - }, - } - - request = SendMessageRequest( - id=str(uuid4()), - params=MessageSendParams(**send_message_payload), - ) - - # Send message - response = await asend_message(a2a_client=a2a_client, request=request) - - # Give async logging time to complete - await asyncio.sleep(1) - - # Print debug info - print("\n=== Logging Validation ===") - print(f"log_success_called: {test_logger.log_success_called}") - print(f"standard_logging_payload: {test_logger.standard_logging_payload}") - print( - f"logged kwargs: {json.dumps(test_logger.logged_kwargs, indent=4, default=str)}" - ) - - # Verify logging was called - assert test_logger.log_success_called is True - assert test_logger.standard_logging_payload is not None - - # Verify standard_logging_payload exists - slp = test_logger.standard_logging_payload - assert slp is not None - - # Get values from standard logging payload - logged_model = ( - slp.get("model") if isinstance(slp, dict) else getattr(slp, "model", None) - ) - logged_provider = ( - slp.get("custom_llm_provider") - if isinstance(slp, dict) - else getattr(slp, "custom_llm_provider", None) - ) - call_type = ( - slp.get("call_type") - if isinstance(slp, dict) - else getattr(slp, "call_type", None) - ) - response_cost = ( - slp.get("response_cost") - if isinstance(slp, dict) - else getattr(slp, "response_cost", None) - ) - - print(f"\n=== Standard Logging Payload Validation ===") - print(f"model: {logged_model}") - print(f"custom_llm_provider: {logged_provider}") - print(f"call_type: {call_type}") - print(f"response_cost: {response_cost}") - - # Verify model and custom_llm_provider are set correctly - assert logged_model is not None, "model should be set" - assert ( - "a2a_agent/" in logged_model - ), f"model should contain 'a2a_agent/', got: {logged_model}" - assert ( - logged_provider == "a2a_agent" - ), f"custom_llm_provider should be 'a2a_agent', got: {logged_provider}" - - # Verify call_type is correct for A2A - assert ( - call_type == "asend_message" - ), f"call_type should be 'asend_message', got: {call_type}" - - # Verify response_cost is set to 0.0 (not None, not an error) - # This confirms the A2A cost calculator is working - assert response_cost is not None, "response_cost should not be None" - assert ( - response_cost == 0.0 - ), f"response_cost should be 0.0 for A2A, got: {response_cost}" - - -@pytest.mark.asyncio -async def test_pydantic_ai_non_streaming(): - """ - Test non-streaming requests to Pydantic AI agents. - - Pydantic AI agents follow A2A protocol but don't support streaming. - This test validates non-streaming requests work correctly. - """ - litellm.turn_on_debug() - from litellm.a2a_protocol import asend_message - - # Build the request - send_message_payload = { - "message": { - "role": "user", - "parts": [ - { - "kind": "text", - "text": "Hello from Pydantic AI test!", - } - ], - "messageId": uuid4().hex, - }, - } - - request = SendMessageRequest( - id=str(uuid4()), - params=MessageSendParams(**send_message_payload), - ) - - # Send message using Pydantic AI provider - response = await asend_message( - request=request, - api_base="http://localhost:9999", - litellm_params={"custom_llm_provider": "pydantic_ai_agents"}, - ) - - # Print response for debugging - print("\n=== Pydantic AI Non-Streaming Response ===") - print(response.model_dump(mode="json", exclude_none=True)) - - # Basic assertions - assert response is not None - assert hasattr(response, "result") - - # Verify result structure - result = response.result - assert result is not None - - # Pydantic AI returns a task with history/artifacts, not a direct message - # Check for either format - result_dict = ( - result - if isinstance(result, dict) - else result.model_dump(mode="python", exclude_none=True) - ) - has_message = "message" in result_dict - has_history = "history" in result_dict - has_artifacts = "artifacts" in result_dict - - assert ( - has_message or has_history or has_artifacts - ), f"Result should contain 'message', 'history', or 'artifacts'. Got: {list(result_dict.keys())}" - - # If it's a task response (Pydantic AI style), verify we got agent response - if has_history: - history = result_dict.get("history", []) - agent_messages = [m for m in history if m.get("role") == "agent"] - assert ( - len(agent_messages) > 0 - ), "Should have at least one agent message in history" - - # Verify agent message has text content - agent_msg = agent_messages[-1] - parts = agent_msg.get("parts", []) - text_parts = [p for p in parts if p.get("kind") == "text"] - assert len(text_parts) > 0, "Agent message should have text content" - print(f"\nAgent response: {text_parts[0].get('text')}") - - -@pytest.mark.asyncio -async def test_pydantic_ai_fake_streaming(): - """ - Test fake streaming for Pydantic AI agents. - - Pydantic AI agents don't support streaming natively. - This test validates that fake streaming works by converting - non-streaming responses into streaming chunks. - """ - litellm.turn_on_debug() - from litellm.a2a_protocol import asend_message_streaming - - # Build the request - from a2a.types import SendStreamingMessageRequest - - send_message_payload = { - "message": { - "role": "user", - "parts": [ - { - "kind": "text", - "text": "Hello from Pydantic AI streaming test!", - } - ], - "messageId": uuid4().hex, - }, - } - - request = SendStreamingMessageRequest( - id=str(uuid4()), - params=MessageSendParams(**send_message_payload), - ) - - # Send streaming message using Pydantic AI provider - print("\n=== Pydantic AI Fake Streaming Response ===") - chunks_received = 0 - task_event_received = False - working_event_received = False - artifact_event_received = False - completed_event_received = False - - async for chunk in asend_message_streaming( - request=request, - api_base="http://localhost:9999", - litellm_params={"custom_llm_provider": "pydantic_ai_agents"}, - ): - chunks_received += 1 - print(f"\nChunk {chunks_received}:") - - # Convert chunk to dict for inspection - chunk_dict = ( - chunk.model_dump(mode="json", exclude_none=True) - if hasattr(chunk, "model_dump") - else chunk - ) - print(json.dumps(chunk_dict, indent=2)) - - # Check event types - result = chunk_dict.get("result", {}) - kind = result.get("kind") - - if kind == "task": - task_event_received = True - elif kind == "status-update": - status = result.get("status", {}) - state = status.get("state") - if state == "working": - working_event_received = True - elif state == "completed": - completed_event_received = True - elif kind == "artifact-update": - artifact_event_received = True - - print(f"\n=== Streaming Summary ===") - print(f"Total chunks received: {chunks_received}") - print(f"Task event received: {task_event_received}") - print(f"Working event received: {working_event_received}") - print(f"Artifact event received: {artifact_event_received}") - print(f"Completed event received: {completed_event_received}") - - # Verify we received chunks - assert chunks_received > 0, "Should receive at least one chunk" - - # Verify all required event types were received - assert task_event_received, "Should receive task event" - assert working_event_received, "Should receive working status event" - assert artifact_event_received, "Should receive artifact update event" - assert completed_event_received, "Should receive completed status event" diff --git a/tests/agent_tests/local_only_agent_tests/test_a2a_completion_bridge.py b/tests/agent_tests/local_only_agent_tests/test_a2a_completion_bridge.py deleted file mode 100644 index fb2f19bde2a..00000000000 --- a/tests/agent_tests/local_only_agent_tests/test_a2a_completion_bridge.py +++ /dev/null @@ -1,287 +0,0 @@ -""" -Test for A2A to LiteLLM Completion Bridge. - -Tests the SDK-level functions that route A2A requests through litellm.acompletion. - -Run with: - pytest tests/agent_tests/test_a2a_completion_bridge.py -v -s - -Prerequisites: - - LangGraph server running on localhost:2024 -""" - -from uuid import uuid4 - -import pytest - - -import litellm -from a2a.types import MessageSendParams, SendMessageRequest, SendStreamingMessageRequest - - -@pytest.mark.asyncio -async def test_a2a_completion_bridge_non_streaming(): - """ - Test non-streaming A2A request via the completion bridge with LangGraph provider. - """ - from litellm.a2a_protocol import asend_message - - litellm.turn_on_debug() - - send_message_payload = { - "message": { - "role": "user", - "parts": [{"kind": "text", "text": "What is 2 + 2?"}], - "messageId": uuid4().hex, - } - } - - request = SendMessageRequest( - id=str(uuid4()), - params=MessageSendParams(**send_message_payload), # type: ignore - ) - - response = await asend_message( - request=request, - api_base="http://localhost:2024", - litellm_params={"custom_llm_provider": "langgraph", "model": "agent"}, - ) - - # Validate response is LiteLLMSendMessageResponse - assert response.jsonrpc == "2.0" - assert response.id is not None - assert response.result is not None - assert response.result.get("kind") == "message" - - message = response.result - assert "role" in message - assert message["role"] == "agent" - assert "parts" in message - assert len(message["parts"]) > 0 - assert message["parts"][0]["kind"] == "text" - assert len(message["parts"][0]["text"]) > 0 - - print(f"Response: {response.model_dump(mode='json', exclude_none=True)}") - - -@pytest.mark.asyncio -async def test_a2a_completion_bridge_streaming(): - """ - Test streaming A2A request via the completion bridge with LangGraph provider. - - Validates proper A2A streaming format with events: - 1. Task event (kind: "task") - Initial task with status "submitted" - 2. Status update (kind: "status-update") - Status "working" - 3. Artifact update (kind: "artifact-update") - Content delivery - 4. Status update (kind: "status-update") - Final "completed" status - """ - from litellm.a2a_protocol import asend_message_streaming - - litellm.turn_on_debug() - - send_message_payload = { - "message": { - "role": "user", - "parts": [{"kind": "text", "text": "Count from 1 to 5."}], - "messageId": uuid4().hex, - } - } - - request = SendStreamingMessageRequest( - id=str(uuid4()), - params=MessageSendParams(**send_message_payload), # type: ignore - ) - - chunks = [] - async for chunk in asend_message_streaming( - request=request, - api_base="http://localhost:2024", - litellm_params={"custom_llm_provider": "langgraph", "model": "agent"}, - ): - chunks.append(chunk) - print(f"Chunk: {chunk}") - - # Validate we received proper A2A streaming events - assert ( - len(chunks) >= 4 - ), f"Expected at least 4 chunks (task, working, artifact, completed), got {len(chunks)}" - - # Validate chunk structure follows A2A spec - for chunk in chunks: - assert "jsonrpc" in chunk - assert chunk["jsonrpc"] == "2.0" - assert "id" in chunk - assert "result" in chunk - - # Validate first chunk is task event - task_chunk = chunks[0] - assert task_chunk["result"]["kind"] == "task", "First chunk should be task event" - assert task_chunk["result"]["status"]["state"] == "submitted" - assert "contextId" in task_chunk["result"] - assert "id" in task_chunk["result"] # task id - assert "history" in task_chunk["result"] - - # Validate second chunk is working status update - working_chunk = chunks[1] - assert ( - working_chunk["result"]["kind"] == "status-update" - ), "Second chunk should be status-update" - assert working_chunk["result"]["status"]["state"] == "working" - assert "taskId" in working_chunk["result"] - assert "contextId" in working_chunk["result"] - assert working_chunk["result"]["final"] is False - - # Validate artifact update chunk - artifact_chunk = chunks[2] - assert ( - artifact_chunk["result"]["kind"] == "artifact-update" - ), "Third chunk should be artifact-update" - assert "artifact" in artifact_chunk["result"] - assert "artifactId" in artifact_chunk["result"]["artifact"] - assert "parts" in artifact_chunk["result"]["artifact"] - assert len(artifact_chunk["result"]["artifact"]["parts"]) > 0 - assert artifact_chunk["result"]["artifact"]["parts"][0]["kind"] == "text" - - # Validate final chunk is completed status update - final_chunk = chunks[-1] - assert ( - final_chunk["result"]["kind"] == "status-update" - ), "Last chunk should be status-update" - assert final_chunk["result"]["status"]["state"] == "completed" - assert final_chunk["result"]["final"] is True - - print(f"Received {len(chunks)} chunks with proper A2A streaming format") - - -@pytest.mark.asyncio -async def test_a2a_completion_bridge_bedrock_agentcore(): - """ - Test A2A request via the completion bridge with Bedrock AgentCore provider. - - Uses the AgentCore runtime ARN to call a hosted agent. - """ - from litellm.a2a_protocol import asend_message_streaming - - litellm.turn_on_debug() - - # Bedrock AgentCore ARN (streaming-capable runtime) - agentcore_arn = "arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC" - - send_message_payload = { - "message": { - "role": "user", - "parts": [ - {"kind": "text", "text": "Explain machine learning in simple terms"} - ], - "messageId": uuid4().hex, - } - } - - request = SendStreamingMessageRequest( - id=str(uuid4()), - params=MessageSendParams(**send_message_payload), # type: ignore - ) - - chunks = [] - async for chunk in asend_message_streaming( - request=request, - api_base=None, # Not needed for Bedrock AgentCore - litellm_params={ - "custom_llm_provider": "bedrock", - "model": f"bedrock/agentcore/{agentcore_arn}", - }, - ): - chunks.append(chunk) - print(f"Chunk: {chunk}") - - # Validate we received proper A2A streaming events - assert len(chunks) >= 4, f"Expected at least 4 chunks, got {len(chunks)}" - - # Validate first chunk is task event - assert chunks[0]["result"]["kind"] == "task" - assert chunks[0]["result"]["status"]["state"] == "submitted" - - # Validate final chunk is completed status - assert chunks[-1]["result"]["kind"] == "status-update" - assert chunks[-1]["result"]["status"]["state"] == "completed" - assert chunks[-1]["result"]["final"] is True - - print(f"Received {len(chunks)} chunks from Bedrock AgentCore") - - -# ============================================================ -# Vertex AI Agent Engine Tests -# ============================================================ - -# Configuration - update these for your Vertex AI Reasoning Engine -VERTEX_AGENT_RESOURCE_NAME = ( - "projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888" -) - - -@pytest.mark.asyncio -async def test_vertex_agent_engine_non_streaming(): - """ - Test non-streaming request to Vertex AI Agent Engine via litellm.acompletion. - - Uses the Reasoning Engine resource ID to call a hosted agent. - """ - - litellm.turn_on_debug() - - # Call via litellm.acompletion with vertex_ai/agent_engine/ prefix - response = await litellm.acompletion( - model=f"vertex_ai/agent_engine/{VERTEX_AGENT_RESOURCE_NAME}", - messages=[{"role": "user", "content": "Hello! What can you do?"}], - stream=False, - ) - - print(f"\n=== Vertex Agent Engine Non-Streaming Response ===") - print(f"Response: {response}") - - # Basic assertions - assert response is not None - assert hasattr(response, "choices") - assert len(response.choices) > 0 - assert response.choices[0].message is not None - assert response.choices[0].message.content is not None - assert len(response.choices[0].message.content) > 0 - - print(f"Agent response: {response.choices[0].message.content[:200]}...") - - -@pytest.mark.asyncio -async def test_vertex_agent_engine_streaming(): - """ - Test streaming request to Vertex AI Agent Engine via litellm.acompletion. - - Uses the Reasoning Engine resource ID to call a hosted agent with streaming. - """ - # litellm.turn_on_debug() - - # Call via litellm.acompletion with streaming - response = await litellm.acompletion( - model=f"vertex_ai/agent_engine/{VERTEX_AGENT_RESOURCE_NAME}", - messages=[{"role": "user", "content": "Hello! What can you do?"}], - stream=True, - ) - - print(f"\n=== Vertex Agent Engine Streaming Response ===") - - chunks = [] - full_content = "" - async for chunk in response: - print(f"Chunk: {chunk}") - # chunks.append(chunk) - # if hasattr(chunk, "choices") and len(chunk.choices) > 0: - # delta = chunk.choices[0].delta - # if hasattr(delta, "content") and delta.content: - # full_content += delta.content - # print(f"Chunk: {delta.content}", end="", flush=True) - - # # print(f"\n\nReceived {len(chunks)} chunks") - # print(f"Full content: {full_content[:200]}...") - - # # Basic assertions - # assert len(chunks) > 0 - # assert len(full_content) > 0 diff --git a/tests/basic_proxy_startup_tests/test_basic_proxy_startup.py b/tests/basic_proxy_startup_tests/test_basic_proxy_startup.py deleted file mode 100644 index b6acfea44f6..00000000000 --- a/tests/basic_proxy_startup_tests/test_basic_proxy_startup.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -This test ensures that the proxy starts and serves requests even with a bad license. - - -in ci/cd config.yml, we set the license to "bad-license" -""" - -import os -import pytest -import aiohttp -from typing import Optional - - -@pytest.mark.asyncio -async def test_health_and_chat_completion(): - """ - Test health endpoints and chat completion: - 1. Check /health/readiness - 2. Check /health/liveness - 3. Make a chat completion call - """ - async with aiohttp.ClientSession() as session: - # Test readiness endpoint - async with session.get("http://0.0.0.0:4000/health/readiness") as response: - assert response.status == 200 - readiness_response = await response.json() - # Accept both "healthy" (new format) and "connected" (legacy format) - # since this test runs against both source builds and packaged installs - assert readiness_response["status"] in ("healthy", "connected") - - # Test liveness endpoint - async with session.get("http://0.0.0.0:4000/health/liveness") as response: - assert response.status == 200 - liveness_response = await response.json() - print("liveness_response", liveness_response) - - # Make a chat completion call - url = "http://0.0.0.0:4000/chat/completions" - headers = { - "Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", - "Content-Type": "application/json", - } - data = { - "model": "gpt-4", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"}, - ], - } - - async with session.post(url, headers=headers, json=data) as response: - assert response.status == 200 - completion_response = await response.json() - assert "choices" in completion_response diff --git a/tests/code_coverage_tests/code_qa_check_tests.py b/tests/code_coverage_tests/code_qa_check_tests.py index 6c620a02522..17e7466ab71 100644 --- a/tests/code_coverage_tests/code_qa_check_tests.py +++ b/tests/code_coverage_tests/code_qa_check_tests.py @@ -174,7 +174,7 @@ def main(): f"This can cause import issues and test failures. Files: {problematic_files}" ) else: - print("✓ No litellm module deletion patterns found in tests/test_litellm or tests/unit.") + print("✓ No litellm module deletion patterns found in tests/unit.") if __name__ == "__main__": diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 9b9206194b4..e7c2e4234f5 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -27,11 +27,11 @@ def get_function_names_from_file(file_path): def get_all_functions_called_in_tests(base_dir): """ Returns a set of function names that are called in test functions - inside 'local_testing' and 'router_unit_test' directories, + inside the unit test directories, specifically in files containing the word 'router'. """ called_functions = set() - test_dirs = ["local_testing", "router_unit_tests", "test_litellm", "unit"] + test_dirs = ["unit"] for test_dir in test_dirs: dir_path = os.path.join(base_dir, test_dir) diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index b03d23fc473..a0059122ff0 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -260,7 +260,7 @@ The semantic header set is `content-type`, `accept`, `anthropic-version`, `anthr Excluded transport and telemetry headers are `host`, `content-length`, `connection`, `accept-encoding`, `user-agent`, `traceparent`, `tracestate`, `x-request-id`, `x-client-request-id` and `x-stainless-*`. Inbound transfer-encoding is unsupported; send JSON with content-length framing. The destination represents host identity and the relay carries original body bytes. Replay does not verify credentials, SDK timeout/retry behavior, transport performance, model availability or stateful remote IDs. Live relay uses original request bytes and header values, never the stored identity -Strict replay harness regression tests live in `tests/code_coverage_tests/test_provider_replay_harness.py`. The CircleCI `provider_replay_harness` job runs them alongside the provider-edge and fixture tests at the root of `tests/e2e_harness/` with `--noconftest -o "pythonpath=tests/e2e tests/e2e_harness"`; they need only synthetic HTTP providers and temporary fixture storage +Strict replay harness regression tests live in `tests/code_coverage_tests/test_provider_replay_harness.py`. The `python` job in `.github/workflows/test-linting.yml` runs them alongside the provider-edge and fixture tests at the root of `tests/e2e_harness/` with `--noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures`; they need only synthetic HTTP providers and temporary fixture storage ## MCP OAuth happy path diff --git a/tests/e2e/migrations/checks.py b/tests/e2e/migrations/checks.py index 4d67dbaa20d..a1748ec34e6 100644 --- a/tests/e2e/migrations/checks.py +++ b/tests/e2e/migrations/checks.py @@ -39,6 +39,25 @@ def assert_completed(database: Database, migration: Migration = COMPLETE) -> Non assert database.query("SELECT id FROM migration_effect") == ((1,),) +@step("Check that migration {migration.name} rolled back lock timeouts and completed exactly once") +def assert_completed_after_lock_timeouts(database: Database, migration: Migration) -> None: + rows: Final = database.query( + "SELECT finished_at IS NOT NULL, rolled_back_at IS NOT NULL, applied_steps_count FROM " + "_prisma_migrations WHERE migration_name = %s ORDER BY started_at", + (migration.name,), + ) + finished: Final = tuple(row for row in rows if row[0]) + rolled_back: Final = tuple(row for row in rows if row[1]) + assert finished == ((True, False, 1),), "Expected exactly one completed migration attempt" + assert rolled_back and all(row == (False, True, 0) for row in rolled_back), ( + "Expected every failed migration attempt to be rolled back" + ) + assert all(is_finished != is_rolled_back for is_finished, is_rolled_back, _ in rows), ( + "Expected every migration attempt to be either completed or rolled back" + ) + assert database.query("SELECT id FROM migration_effect") == ((1,),) + + @step("Apply the test migration by hand and record it in _prisma_migrations") def confirmed_history(database: Database) -> str: database.execute(COMPLETE_SQL) diff --git a/tests/e2e/migrations/containers.py b/tests/e2e/migrations/containers.py index a05941373be..c606199553f 100644 --- a/tests/e2e/migrations/containers.py +++ b/tests/e2e/migrations/containers.py @@ -41,6 +41,7 @@ def until(description: str, condition: Callable[[], bool], seconds: float = 150) @dataclass(frozen=True, slots=True) class Replica: name: str + identity: str transport: HttpTransport output: Path @@ -171,7 +172,7 @@ class Containers: "model_list: []\ngeneral_settings:\n master_key: os.environ/LITELLM_MASTER_KEY\n" ) env: Final = { - "DATABASE_URL": prisma_url(database.container_url, database.schema), + "DATABASE_URL": prisma_url(database.container_url, database.schema, application_name=name), "LITELLM_MASTER_KEY": MASTER_KEY, "LITELLM_SALT_KEY": MASTER_KEY, "LITELLM_LOCAL_MODEL_COST_MAP": "True", @@ -209,7 +210,12 @@ class Containers: "4000", ) port: Final = int(docker("port", name, "4000/tcp").rsplit(":", 1)[1]) - replica: Final = Replica(name, HttpTransport(f"http://127.0.0.1:{port}", MASTER_KEY, 15), directory) + replica: Final = Replica( + name=name, + identity=name, + transport=HttpTransport(f"http://127.0.0.1:{port}", MASTER_KEY, 15), + output=directory, + ) yield replica finally: try: diff --git a/tests/e2e/migrations/database.py b/tests/e2e/migrations/database.py index 929a6aee817..9bf03a216c4 100644 --- a/tests/e2e/migrations/database.py +++ b/tests/e2e/migrations/database.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Generator from contextlib import contextmanager from dataclasses import dataclass +from itertools import chain from typing import Final, LiteralString from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from uuid import uuid4 @@ -24,10 +25,19 @@ def connect_url(url: str, name: str) -> str: return urlunsplit(urlsplit(url)._replace(path=f"/{name}", query="")) -def prisma_url(url: str, schema: str) -> str: +def prisma_url(url: str, schema: str, application_name: str | None = None) -> str: parsed: Final = urlsplit(url) - query: Final = tuple((key, value) for key, value in parse_qsl(parsed.query) if key != "schema") - return urlunsplit(parsed._replace(query=urlencode((*query, ("schema", schema))))) + query: Final = tuple( + (key, value) + for key, value in parse_qsl(parsed.query) + if key != "schema" and (application_name is None or key != "application_name") + ) + parameters: Final = query + ( + (("schema", schema),) + if application_name is None + else (("schema", schema), ("application_name", application_name)) + ) + return urlunsplit(parsed._replace(query=urlencode(parameters))) @dataclass(frozen=True, slots=True) @@ -79,6 +89,27 @@ class Database: (key >> 32, key & 0xFFFFFFFF), ) + @step("List replica identities waiting on any of the advisory locks") + def waiting_identities(self, keys: tuple[int, ...]) -> frozenset[str]: + if not keys: + return frozenset() + key_pairs: Final = tuple((key >> 32, key & 0xFFFFFFFF) for key in keys) + key_predicates: Final = sql.SQL(" OR ").join( + sql.SQL("(waiting_lock.classid = %s AND waiting_lock.objid = %s)") for _ in key_pairs + ) + parameters: Final = tuple(chain.from_iterable(key_pairs)) + statement: Final = sql.SQL( + "SELECT DISTINCT activity.application_name " + "FROM pg_stat_activity AS activity " + "JOIN pg_locks AS waiting_lock ON waiting_lock.pid = activity.pid " + "WHERE activity.datname = current_database() " + "AND waiting_lock.locktype = 'advisory' AND NOT waiting_lock.granted " + "AND waiting_lock.database = (SELECT oid FROM pg_database WHERE datname = current_database()) " + "AND ({})" + ).format(key_predicates) + identities: Final = self.query(statement, parameters) + return frozenset(row[0] for row in identities if isinstance(row[0], str)) + @step("Hold an advisory lock on the test database") @contextmanager def lock(self, key: int = GATE_KEY) -> Generator[None]: diff --git a/tests/e2e/migrations/test_startup.py b/tests/e2e/migrations/test_startup.py index ff7112c3881..856192937e2 100644 --- a/tests/e2e/migrations/test_startup.py +++ b/tests/e2e/migrations/test_startup.py @@ -1,17 +1,31 @@ from contextlib import ExitStack +from itertools import accumulate, repeat from typing import Final import pytest -from .checks import COMPLETE, FATAL, GATED, assert_completed, start_replicas +from .checks import ( + COMPLETE, + FATAL, + GATED, + assert_completed, + assert_completed_after_lock_timeouts, + start_replicas, +) from .containers import Containers, failed, ready, until, waiting -from .database import PRISMA_LOCK, Database, Databases, restricted_user +from .database import GATE_KEY, PRISMA_LOCK, Database, Databases, restricted_user from .startup_models import Migration from e2e_metadata import Domain, Subject, meta pytestmark: Final = [pytest.mark.e2e, pytest.mark.migration_startup] +def union_waiting_identities( + observed: frozenset[str], snapshot: frozenset[str] +) -> frozenset[str]: + return observed | snapshot + + class TestMigrationStartup: @pytest.mark.parametrize("replicas,v2", ((1, True), (3, True), (1, False))) @meta( @@ -42,16 +56,34 @@ class TestMigrationStartup: domain=Domain.DB, ) ) - def test_waiters_survive_prolonged_contention(self, containers: Containers, database: Database) -> None: + def test_waiters_retry_lock_timeouts_until_contention_clears( + self, containers: Containers, database: Database + ) -> None: with ExitStack() as stack: with database.lock(): owner: Final = stack.enter_context(containers.start(database, (GATED,))) until("owner blocked in migration SQL", lambda: bool(database.blocked())) followers: Final = start_replicas(stack, containers, database, (GATED,), count=2) - until("both followers attempted Prisma locking", lambda: len(database.blocked(PRISMA_LOCK)) == 2) - waiting((owner, *followers), 120) + waiting_identities: Final = accumulate( + (database.waiting_identities((PRISMA_LOCK, GATE_KEY)) for _ in repeat(None)), + union_waiting_identities, + initial=frozenset(), + ) + follower_identities: Final = frozenset(follower.identity for follower in followers) + until( + "both followers entered migration locking", + lambda: follower_identities <= next(waiting_identities), + ) + until( + "a replica logged a migration lock-timeout retry", + lambda: any( + "timed out waiting for a table lock" in replica.logs() + for replica in (owner, *followers) + ), + ) + waiting((owner, *followers), 20) ready((owner, *followers), database) - assert_completed(database, GATED) + assert_completed_after_lock_timeouts(database, GATED) @meta( Subject( @@ -72,6 +104,27 @@ class TestMigrationStartup: ready((restarted,), database) assert_completed(database) + @meta( + Subject( + domain=Domain.DB, + ) + ) + def test_lock_timeout_exhaustion_then_restart(self, containers: Containers, database: Database) -> None: + with database.lock(): + with containers.start( + database, (GATED,), environment={"LITELLM_MIGRATION_DDL_LOCK_TIMEOUT": "2"} + ) as replica: + until("owner blocked in migration SQL", lambda: bool(database.blocked())) + failed((replica,), "attempts that made no progress") + assert not database.exists("migration_effect") + assert database.query( + "SELECT count(*) FROM _prisma_migrations WHERE migration_name = %s AND finished_at IS NOT NULL", + (GATED.name,), + ) == ((0,),) + with containers.start(database, (GATED,)) as restarted: + ready((restarted,), database) + assert_completed_after_lock_timeouts(database, GATED) + @meta( Subject( domain=Domain.DB, diff --git a/tests/e2e/quota_management/spend_tracking/SPEND_TRACKING_COVERAGE_MATRIX.md b/tests/e2e/quota_management/spend_tracking/SPEND_TRACKING_COVERAGE_MATRIX.md index a86f32f1837..48fed0e9047 100644 --- a/tests/e2e/quota_management/spend_tracking/SPEND_TRACKING_COVERAGE_MATRIX.md +++ b/tests/e2e/quota_management/spend_tracking/SPEND_TRACKING_COVERAGE_MATRIX.md @@ -32,7 +32,7 @@ proxy + SpendLogs rows. Status: `covered` / `partial` / `gap`. | Chat (non-stream) | `test_cost_calculator.py`, `local_testing/test_completion_cost.py` | covered | yes (`test_chat_completion_writes_nonzero_spend_row`) | | Chat (streaming) | `test_streaming_interrupt_spend_tracking.py` | partial | yes (`test_streaming_chat_completion_tracks_spend`) | | Embedding | `test_cost_calculator.py` (#29956) | partial | yes (`test_embedding_writes_nonzero_spend_row`) | -| Pass-through (gemini/anthropic) | `pass_through_tests/*.test.js` + `llm_translation/` suite | covered | yes (llm_translation suite) | +| Pass-through (Gemini/Vertex) | `tests/integration/observability/test_passthrough_upstream_error_visibility.py`, `tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py`, and `tests/e2e/llm_translation/` | covered | yes (integration and e2e suites) | | Image / audio / rerank / responses / realtime | per-provider unit cost tests | partial/gap | gap | ## Entity spend aggregation diff --git a/tests/e2e/ui/migration.serverRootPath.config.ts b/tests/e2e/ui/migration.serverRootPath.config.ts index dc83f3d6584..627353aab45 100644 --- a/tests/e2e/ui/migration.serverRootPath.config.ts +++ b/tests/e2e/ui/migration.serverRootPath.config.ts @@ -12,7 +12,8 @@ export default defineConfig({ testMatch: ["migratedPages.spec.ts"], fullyParallel: true, forbidOnly: !!process.env.CI, - retries: process.env.CI ? 2 : 0, + retries: 0, + grep: /navigates directly between two migrated pages/, workers: process.env.CI ? 1 : undefined, reporter: "list", use: { diff --git a/tests/e2e/ui/tests/logs/logsFilters.spec.ts b/tests/e2e/ui/tests/logs/logsFilters.spec.ts index 7174f339296..9d5ea9cea84 100644 --- a/tests/e2e/ui/tests/logs/logsFilters.spec.ts +++ b/tests/e2e/ui/tests/logs/logsFilters.spec.ts @@ -143,10 +143,12 @@ test.describe("Logs page filters", () => { await applyComboboxFilter(page, drawer, "Search a key alias", mine.alias!); await expect(requestLogsRows(page).filter({ hasText: theirRequestId })).toHaveCount(0, { timeout: 30_000 }); - // A filter you cannot clear is a page that looks empty forever, which is how it reads to a user. await page.getByRole("button", { name: "Reset Filters" }).filter({ visible: true }).click(); + const search = visibleTestId(page, "datatable-search"); + await search.fill(theirRequestId); await expect(requestLogsRows(page).filter({ hasText: theirRequestId })).toHaveCount(1, { timeout: 30_000 }); - await expect(requestLogsRows(page).filter({ hasText: myRequestId })).toHaveCount(1, { timeout: 10_000 }); + await search.fill(myRequestId); + await expect(requestLogsRows(page).filter({ hasText: myRequestId })).toHaveCount(1, { timeout: 30_000 }); }); }); diff --git a/tests/e2e/ui/tests/proxy-admin/keyBudgetWindow.spec.ts b/tests/e2e/ui/tests/proxy-admin/keyBudgetWindow.spec.ts index 4e4d0a395c3..50850a5cd58 100644 --- a/tests/e2e/ui/tests/proxy-admin/keyBudgetWindow.spec.ts +++ b/tests/e2e/ui/tests/proxy-admin/keyBudgetWindow.spec.ts @@ -24,6 +24,8 @@ const test = base.extend<{ scopedKey: ScopedKey }>({ }); test.describe("Proxy Admin - Key budget window", () => { + // TODO: drop once https://github.com/BerriAI/litellm/pull/41350 lands, a background refresh can wipe the edit draft + test.describe.configure({ retries: 2 }); test.use({ storageState: ADMIN_STORAGE_PATH }); test("a monthly spend cap survives a reload, and clearing the window keeps the cap", async ({ page, scopedKey }) => { diff --git a/tests/e2e_harness/AGENTS.md b/tests/e2e_harness/AGENTS.md index a3e1794ce4a..ef4a11802c9 100644 --- a/tests/e2e_harness/AGENTS.md +++ b/tests/e2e_harness/AGENTS.md @@ -14,4 +14,4 @@ LITELLM_MASTER_KEY=sk-harness uv run pytest tests/e2e_harness Rules: no `e2e` marker and no `@meta`, since nothing here drives the proxy. `@pytest.mark.covers` only where the test proves the collector or the JUnit properties read it. Inputs via arguments or env vars (setting an env var through pytest's `monkeypatch` fixture is fine, patching a function, class or module is not). And the same typing bar as the suite, `make lint-e2e-basedpyright` covers this folder and allows zero errors. The raw HTTP client ban (`tests/code_coverage_tests/check_e2e_no_raw_requests.py`) applies here too -CI: the `python` job in `.github/workflows/test-linting.yml` runs this folder whenever anything under `tests/e2e/` (except `ui/`) or `tests/e2e_harness/` changes, with the `claude` CLI installed. The CircleCI `provider_replay_harness` job also runs the provider-edge and fixture tests at the root of this folder next to `tests/code_coverage_tests/test_provider_replay_harness.py`, which imports helpers from `test_provider_edge.py` +CI: the `python` job in `.github/workflows/test-linting.yml` runs this folder whenever anything under `tests/e2e/` (except `ui/`) or `tests/e2e_harness/` changes, with the `claude` CLI installed. It also runs `tests/code_coverage_tests/test_provider_replay_harness.py` and `tests/code_coverage_tests/test_provider_cache.py` with `--noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures`; the replay test imports helpers from `test_provider_edge.py` diff --git a/tests/integration/_support/actors.py b/tests/integration/_support/actors.py new file mode 100644 index 00000000000..6a1e53bda31 --- /dev/null +++ b/tests/integration/_support/actors.py @@ -0,0 +1,443 @@ +import uuid +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence +from contextlib import asynccontextmanager, contextmanager +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +import yaml +from prisma import Prisma +from pydantic import JsonValue + +from litellm.proxy.utils import PrismaClient, hash_token +from tests.integration._support.client import Gateway, Scenario, string_value +from tests.integration._support.prisma_clients import connected_proxy_prisma +from tests.integration._support.process import owned_proxy + +TEAM_ALPHA: Final = "behavior-pin-team-alpha" +TEAM_BETA: Final = "behavior-pin-team-beta" +TEAM_GAMMA: Final = "behavior-pin-team-gamma" +ORG_A: Final = "behavior-pin-org-a" +ORG_B: Final = "behavior-pin-org-b" + + +class Actor(str, Enum): + PROXY_ADMIN = "proxy_admin" + ORG_ADMIN = "org_admin" + TEAM_ADMIN = "team_admin" + INTERNAL_USER = "internal_user" + OWNER = "owner" + UNRELATED_SAME_ORG = "unrelated_same_org" + CROSS_ORG_USER = "cross_org_user" + SERVICE_ACCOUNT = "service_account" + ORG_B_ADMIN = "org_b_admin" + + +@dataclass(frozen=True, slots=True) +class SeededKey: + user_id: str + cleartext: str + hashed: str + + +@dataclass(frozen=True, slots=True) +class ActorSet: + org_a_id: str + org_b_id: str + team_alpha_id: str + team_beta_id: str + team_gamma_id: str + keys: Mapping[Actor, SeededKey] + + +@dataclass(frozen=True, slots=True) +class ScratchResources: + prefix: str + + def tag(self, suffix: str = "") -> str: + return f"{self.prefix}-{suffix}" if suffix else self.prefix + + +@dataclass(frozen=True, slots=True) +class GatewayClient: + gateway: Gateway + replacements: Mapping[str, str] + + async def request( + self, + method: str, + path: str, + *, + headers: Mapping[str, str] | None = None, + json: Mapping[str, JsonValue] | None = None, + params: Mapping[str, str] | None = None, + ) -> httpx.Response: + resolved_path: Final = _replace_text(path, self.replacements) + resolved_body: Final = _replace_body(json, self.replacements) if json is not None else None + return self.gateway.request(method, resolved_path, resolved_body, headers=headers, params=params) + + async def post( + self, + path: str, + *, + headers: Mapping[str, str] | None = None, + json: Mapping[str, JsonValue] | None = None, + ) -> httpx.Response: + return await self.request("POST", path, headers=headers, json=json) + + async def get( + self, + path: str, + *, + headers: Mapping[str, str] | None = None, + params: Mapping[str, str] | None = None, + ) -> httpx.Response: + return await self.request("GET", path, headers=headers, params=params) + + async def put( + self, + path: str, + *, + headers: Mapping[str, str] | None = None, + json: Mapping[str, JsonValue] | None = None, + ) -> httpx.Response: + return await self.request("PUT", path, headers=headers, json=json) + + async def delete( + self, + path: str, + *, + headers: Mapping[str, str] | None = None, + json: Mapping[str, JsonValue] | None = None, + ) -> httpx.Response: + return await self.request("DELETE", path, headers=headers, json=json) + + +@contextmanager +def owned_settings_client( + client: GatewayClient, directory: Path, litellm_settings: Mapping[str, JsonValue] +) -> Iterator[GatewayClient]: + base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config: Final = {**base, "litellm_settings": {**base["litellm_settings"], **litellm_settings}} + path: Final = directory / "owned-settings.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(client.gateway, directory, {}, config=path) as candidate: + yield GatewayClient(candidate, client.replacements) + + +@dataclass(frozen=True, slots=True) +class ManagementDatabase: + prisma: PrismaClient + gateway: Gateway + + @property + def db(self) -> Prisma: + return self.prisma.db + + +def _replace_text(value: str, replacements: Mapping[str, str]) -> str: + return value.replace(TEAM_ALPHA, replacements[TEAM_ALPHA]).replace( + TEAM_BETA, replacements[TEAM_BETA] + ).replace(TEAM_GAMMA, replacements[TEAM_GAMMA]).replace(ORG_A, replacements[ORG_A]).replace( + ORG_B, replacements[ORG_B] + ) + + +def _replace_body( + value: JsonValue, replacements: Mapping[str, str] +) -> JsonValue: + if isinstance(value, str): + return _replace_text(value, replacements) + if isinstance(value, list): + return [_replace_body(item, replacements) for item in value] + if isinstance(value, dict): + return {key: _replace_body(item, replacements) for key, item in value.items()} + return value + + +def _seeded_key( + scenario: Scenario, + actor: Actor, + user_id: str, + *, + team_id: str | None = None, + organization_id: str | None = None, +) -> SeededKey: + token: Final = scenario.key( + key_alias=f"integration-{actor.value}-{uuid.uuid4().hex}", + user_id=user_id, + team_id=team_id, + organization_id=organization_id, + ) + return SeededKey(user_id=user_id, cleartext=token, hashed=hash_token(token)) + + +async def create_actor_set( + gateway: Gateway, scenario: Scenario, db: ManagementDatabase +) -> ActorSet: + run_id: Final = uuid.uuid4().hex + org_a_id: Final = scenario.organization(organization_alias=f"alpha-{run_id}") + org_b_id: Final = scenario.organization(organization_alias=f"beta-{run_id}") + proxy_admin_id: Final = scenario.user(user_role="proxy_admin") + org_admin_a_id: Final = scenario.org_member(org_a_id, "org_admin") + org_admin_b_id: Final = scenario.org_member(org_b_id, "org_admin") + await db.db.litellm_usertable.update( + where={"user_id": org_admin_a_id}, data={"user_role": "org_admin"} + ) + await db.db.litellm_usertable.update( + where={"user_id": org_admin_b_id}, data={"user_role": "org_admin"} + ) + team_alpha_id: Final = scenario.team( + team_id=f"integration-{run_id}-alpha", + team_alias=f"alpha-{run_id}", + organization_id=org_a_id, + team_member_permissions=["/key/health"], + ) + team_beta_id: Final = scenario.team( + team_id=f"integration-{run_id}-beta", + team_alias=f"beta-{run_id}", + organization_id=org_b_id, + team_member_permissions=["/key/health"], + ) + team_gamma_id: Final = scenario.team( + team_id=f"integration-{run_id}-gamma", + team_alias=f"gamma-{run_id}", + organization_id=org_a_id, + team_member_permissions=["/key/health"], + ) + team_admin_id: Final = scenario.member(team_alpha_id, role="admin") + internal_user_id: Final = scenario.member(team_alpha_id) + owner_id: Final = scenario.member(team_alpha_id) + unrelated_id: Final = scenario.member(team_alpha_id) + service_account_id: Final = scenario.member(team_alpha_id) + cross_org_id: Final = scenario.member(team_beta_id) + keys: Final[Mapping[Actor, SeededKey]] = MappingProxyType( + { + Actor.PROXY_ADMIN: _seeded_key(scenario, Actor.PROXY_ADMIN, proxy_admin_id), + Actor.ORG_ADMIN: _seeded_key( + scenario, Actor.ORG_ADMIN, org_admin_a_id, organization_id=org_a_id + ), + Actor.TEAM_ADMIN: _seeded_key( + scenario, Actor.TEAM_ADMIN, team_admin_id, team_id=team_alpha_id, organization_id=org_a_id + ), + Actor.INTERNAL_USER: _seeded_key( + scenario, Actor.INTERNAL_USER, internal_user_id, team_id=team_alpha_id, organization_id=org_a_id + ), + Actor.OWNER: _seeded_key( + scenario, Actor.OWNER, owner_id, team_id=team_alpha_id, organization_id=org_a_id + ), + Actor.UNRELATED_SAME_ORG: _seeded_key( + scenario, Actor.UNRELATED_SAME_ORG, unrelated_id, team_id=team_alpha_id, organization_id=org_a_id + ), + Actor.CROSS_ORG_USER: _seeded_key( + scenario, Actor.CROSS_ORG_USER, cross_org_id, team_id=team_beta_id, organization_id=org_b_id + ), + Actor.SERVICE_ACCOUNT: _seeded_key( + scenario, Actor.SERVICE_ACCOUNT, service_account_id, team_id=team_alpha_id, organization_id=org_a_id + ), + Actor.ORG_B_ADMIN: _seeded_key( + scenario, Actor.ORG_B_ADMIN, org_admin_b_id, organization_id=org_b_id + ), + } + ) + return ActorSet(org_a_id, org_b_id, team_alpha_id, team_beta_id, team_gamma_id, keys) + + +def create_gateway_client(gateway: Gateway, actors: ActorSet) -> GatewayClient: + replacements: Final[Mapping[str, str]] = MappingProxyType( + { + TEAM_ALPHA: actors.team_alpha_id, + TEAM_BETA: actors.team_beta_id, + TEAM_GAMMA: actors.team_gamma_id, + ORG_A: actors.org_a_id, + ORG_B: actors.org_b_id, + } + ) + return GatewayClient(gateway, replacements) + + +async def create_scratch_key( + gateway_client: GatewayClient, + seeder_cleartext: str, + scratch_prefix: str, + *, + user_id: str, + team_id: str | None = None, + organization_id: str | None = None, + key_alias: str | None = None, +) -> str: + body: Final[dict[str, JsonValue]] = { + "key_alias": key_alias or scratch_prefix, + "user_id": user_id, + **({"team_id": team_id} if team_id is not None else {}), + **({"organization_id": organization_id} if organization_id is not None else {}), + } + created: Final = gateway_client.gateway.post( + "/key/generate", body, key=seeder_cleartext + ) + return string_value(created["key"]) + + +async def create_scratch_team( + db: ManagementDatabase, + team_id: str, + *, + organization_id: str | None = None, + admin_user_ids: Sequence[str] = (), + member_user_ids: Sequence[str] = (), + team_member_permissions: Sequence[str] | None = None, + models: Sequence[str] | None = None, + max_budget: float | None = None, + tpm_limit: int | None = None, + rpm_limit: int | None = None, + metadata: Mapping[str, JsonValue] | None = None, +) -> str: + admins: Final = list(admin_user_ids) + members: Final = list(member_user_ids) + members_with_roles: Final = [ + {"user_id": identity, "role": "admin"} for identity in admins + ] + [{"user_id": identity, "role": "user"} for identity in members] + fields: Final[dict[str, JsonValue]] = { + "team_id": team_id, + "team_alias": team_id, + "admins": admins, + "members": admins + members, + "members_with_roles": members_with_roles, + **({"organization_id": organization_id} if organization_id is not None else {}), + **( + {"team_member_permissions": list(team_member_permissions)} + if team_member_permissions is not None + else {} + ), + **({"models": list(models)} if models is not None else {}), + **({"max_budget": max_budget} if max_budget is not None else {}), + **({"tpm_limit": tpm_limit} if tpm_limit is not None else {}), + **({"rpm_limit": rpm_limit} if rpm_limit is not None else {}), + **({"metadata": dict(metadata)} if metadata is not None else {}), + } + created: Final = db.gateway.post("/team/new", fields) + return string_value(created["team_id"]) + + +async def create_scratch_user( + db: ManagementDatabase, + scratch_prefix: str, + *, + suffix: str, + user_email: str | None = None, +) -> str: + user_id: Final = f"{scratch_prefix}-{suffix}" + fields: Final[dict[str, JsonValue]] = { + "user_id": user_id, + "user_role": "internal_user", + "auto_create_key": False, + **({"user_email": user_email} if user_email is not None else {}), + } + created: Final = db.gateway.post("/user/new", fields) + return string_value(created["user_id"]) + + +async def create_scratch_org( + db: ManagementDatabase, + scratch_prefix: str, + *, + max_budget: float | None = None, + tpm_limit: int | None = None, + rpm_limit: int | None = None, + models: Sequence[str] | None = None, + metadata: Mapping[str, JsonValue] | None = None, + suffix: str = "org", +) -> str: + organization_id: Final = f"{scratch_prefix}-{suffix}" + budget_fields: Final[dict[str, JsonValue]] = { + "budget_id": f"{organization_id}-budget", + **({"max_budget": max_budget} if max_budget is not None else {}), + **({"tpm_limit": tpm_limit} if tpm_limit is not None else {}), + **({"rpm_limit": rpm_limit} if rpm_limit is not None else {}), + } + budget: Final = db.gateway.post("/budget/new", budget_fields) + budget_id: Final = string_value(budget["budget_id"]) + fields: Final[dict[str, JsonValue]] = { + "organization_id": organization_id, + "organization_alias": organization_id, + "budget_id": budget_id, + **({"models": list(models)} if models is not None else {}), + **({"metadata": dict(metadata)} if metadata is not None else {}), + } + created: Final = db.gateway.post("/organization/new", fields) + return string_value(created["organization_id"]) + + +async def create_scratch_actor( + db: ManagementDatabase, + scratch_prefix: str, + *, + user_role: str, + org_admin_of: Sequence[str] = (), + organization_id: str | None = None, + suffix: str = "actor", +) -> SeededKey: + api_user_role: Final = ( + user_role + if user_role in {"proxy_admin", "proxy_admin_viewer", "internal_user", "internal_user_viewer"} + else "internal_user" + ) + created_user: Final = db.gateway.post( + "/user/new", + { + "user_id": f"{scratch_prefix}-{suffix}", + "user_role": api_user_role, + "auto_create_key": False, + **({"organization_id": organization_id} if organization_id is not None else {}), + }, + ) + user_id: Final = string_value(created_user["user_id"]) + token: Final = string_value( + db.gateway.post( + "/key/generate", + {"user_id": user_id, "key_alias": f"{scratch_prefix}-{suffix}-key"}, + )["key"] + ) + tuple( + db.gateway.post( + "/organization/member_add", + {"organization_id": org_id, "member": {"user_id": user_id, "role": "org_admin"}}, + ) + for org_id in org_admin_of + ) + return SeededKey(user_id, token, hash_token(token)) + + +@asynccontextmanager +async def management_database(gateway: Gateway) -> AsyncIterator[ManagementDatabase]: + async with connected_proxy_prisma() as client: + yield ManagementDatabase(client, gateway) + + +@asynccontextmanager +async def scratch_resources(db: ManagementDatabase) -> AsyncIterator[ScratchResources]: + resource: Final = ScratchResources(f"integration-{uuid.uuid4().hex}") + yield resource + prefix: Final = resource.prefix + await db.db.litellm_verificationtoken.delete_many( + where={"OR": [{"key_alias": {"startswith": prefix}}, {"key_name": {"startswith": prefix}}]} + ) + await db.db.litellm_teammembership.delete_many(where={"team_id": {"startswith": prefix}}) + await db.db.litellm_organizationmembership.delete_many(where={"user_id": {"startswith": prefix}}) + await db.db.litellm_teamtable.delete_many(where={"team_id": {"startswith": prefix}}) + await db.db.litellm_usertable.delete_many( + where={"OR": [{"user_id": {"startswith": prefix}}, {"user_email": {"startswith": prefix}}]} + ) + await db.db.litellm_organizationtable.delete_many(where={"organization_id": {"startswith": prefix}}) + await db.db.litellm_budgettable.delete_many(where={"budget_id": {"startswith": prefix}}) + users: Final = await db.db.litellm_usertable.find_many(where={"teams": {"isEmpty": False}}) + for user in users: + teams: Final = [team_id for team_id in user.teams if not team_id.startswith(prefix)] + if teams != list(user.teams): + await db.db.litellm_usertable.update( + where={"user_id": user.user_id}, + data={"teams": {"set": teams}}, + ) diff --git a/tests/integration/_support/prisma_clients.py b/tests/integration/_support/prisma_clients.py new file mode 100644 index 00000000000..5f765e2a6db --- /dev/null +++ b/tests/integration/_support/prisma_clients.py @@ -0,0 +1,35 @@ +import os +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Final + +from prisma import Prisma +from prisma.types import DatasourceOverride + +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import PrismaClient, ProxyLogging + + +@asynccontextmanager +async def connected_prisma(database_url: str | None = None) -> AsyncIterator[Prisma]: + client: Final = Prisma( + datasource=DatasourceOverride(url=database_url) + ) if database_url is not None else Prisma() + await client.connect() + try: + yield client + finally: + await client.disconnect() + + +@asynccontextmanager +async def connected_proxy_prisma(database_url: str | None = None) -> AsyncIterator[PrismaClient]: + client: Final = PrismaClient( + database_url=database_url or os.environ["DATABASE_URL"], + proxy_logging_obj=ProxyLogging(UserApiKeyCache()), + ) + await client.connect() + try: + yield client + finally: + await client.disconnect() diff --git a/tests/proxy_behavior/__init__.py b/tests/integration/authorization/__init__.py similarity index 100% rename from tests/proxy_behavior/__init__.py rename to tests/integration/authorization/__init__.py diff --git a/tests/proxy_behavior/auth/test_auth_object_prefetch.py b/tests/integration/authorization/test_auth_object_prefetch.py similarity index 96% rename from tests/proxy_behavior/auth/test_auth_object_prefetch.py rename to tests/integration/authorization/test_auth_object_prefetch.py index 2d3b6da2a45..b37399b5338 100644 --- a/tests/proxy_behavior/auth/test_auth_object_prefetch.py +++ b/tests/integration/authorization/test_auth_object_prefetch.py @@ -3,10 +3,12 @@ team and hand the getters rows they validate. The per-regime round-trip counts a tests/unit/proxy/auth/test_auth_object_prefetch.py.""" import json +from collections.abc import AsyncIterator from unittest.mock import AsyncMock, MagicMock from uuid import uuid4 import pytest +import pytest_asyncio from litellm.caching.in_memory_cache import InMemoryCache from litellm.proxy.auth.auth_checks import ( @@ -18,10 +20,18 @@ from litellm.proxy.auth.auth_checks import ( from litellm.proxy.auth.auth_object_prefetch import AuthObjectRefs, prefetch_auth_objects from litellm.proxy.auth.team_grants import team_model_aliases from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import PrismaClient +from tests.integration._support.prisma_clients import connected_proxy_prisma pytestmark = pytest.mark.asyncio(loop_scope="session") +@pytest_asyncio.fixture(scope="module", loop_scope="session") +async def prisma() -> AsyncIterator[PrismaClient]: + async with connected_proxy_prisma() as client: + yield client + + def _frozen_cache() -> UserApiKeyCache: """The org entries carry a 5s TTL; a frozen clock keeps a slow first call from expiring them mid-test.""" return UserApiKeyCache(in_memory_cache=InMemoryCache(clock=lambda: 1_000_000.0), redis_cache=None) diff --git a/tests/load_tests/test_granian_admission_saturation.py b/tests/integration/compatibility/test_granian_admission_saturation.py similarity index 56% rename from tests/load_tests/test_granian_admission_saturation.py rename to tests/integration/compatibility/test_granian_admission_saturation.py index b55d75ee9f1..9f7cec44d89 100644 --- a/tests/load_tests/test_granian_admission_saturation.py +++ b/tests/integration/compatibility/test_granian_admission_saturation.py @@ -12,10 +12,9 @@ import pytest from tests._master_key import MASTER_KEY -pytestmark = pytest.mark.skipif( - os.environ.get("LITELLM_RUN_SATURATION_BENCHMARK") != "1", - reason="set LITELLM_RUN_SATURATION_BENCHMARK=1 to run the saturation benchmark", -) +MAX_IN_FLIGHT: Final = 8 +MAX_QUEUED: Final = 8 +REQUESTS: Final = 200 def _free_port() -> int: @@ -24,27 +23,24 @@ def _free_port() -> int: return int(listener.getsockname()[1]) -def _percentile(values: list[float], percentile: float) -> float: - return sorted(values)[min(int(len(values) * percentile), len(values) - 1)] - - @pytest.mark.asyncio async def test_granian_admission_control_saturation(tmp_path: Path) -> None: fake_port: Final = _free_port() proxy_port: Final = _free_port() - fake_script: Final = Path(__file__).parents[1] / "_fake_openai_endpoint_server.py" + fake_script: Final = Path(__file__).parents[2] / "_fake_openai_endpoint_server.py" config_path: Final = tmp_path / "saturation_config.yaml" - proxy_env: Final = {**os.environ, "LITELLM_MASTER_KEY": MASTER_KEY} + proxy_env: Final = {**{k: v for k, v in os.environ.items() if k != "DATABASE_URL"}, "LITELLM_MASTER_KEY": MASTER_KEY} config_path.write_text( f"""model_list: - model_name: slow-endpoint litellm_params: model: openai/slow-endpoint api_base: http://127.0.0.1:{fake_port}/v1 + api_key: fake-upstream-key general_settings: master_key: os.environ/LITELLM_MASTER_KEY - max_in_flight_requests_per_worker: 8 - max_queued_requests_per_worker: 8 + max_in_flight_requests_per_worker: {MAX_IN_FLIGHT} + max_queued_requests_per_worker: {MAX_QUEUED} admission_queue_timeout_seconds: 0.5 """ ) @@ -85,22 +81,7 @@ general_settings: else: raise AssertionError("Granian proxy did not become healthy") - liveness_latencies: Final[list[float]] = [] - stop_sampling: Final = asyncio.Event() - - async def sample_liveness() -> None: - while not stop_sampling.is_set(): - start: Final = time.perf_counter() - try: - response = await client.get("/health/liveliness", timeout=2) - response.raise_for_status() - liveness_latencies.append(time.perf_counter() - start) - except httpx.HTTPError: - pass - await asyncio.sleep(0.05) - - async def send_completion() -> tuple[int, float, bool]: - start: Final = time.perf_counter() + async def send_completion() -> tuple[int, bool]: response = await client.post( "/chat/completions", headers={"Authorization": f"Bearer {MASTER_KEY}"}, @@ -110,34 +91,16 @@ general_settings: }, timeout=10, ) - return response.status_code, time.perf_counter() - start, "retry-after" in response.headers + return response.status_code, "retry-after" in response.headers - sampler: Final = asyncio.create_task(sample_liveness()) - results: Final = await asyncio.gather(*(send_completion() for _ in range(200))) - stop_sampling.set() - await sampler + results: Final = await asyncio.gather(*(send_completion() for _ in range(REQUESTS))) statuses: Final = [result[0] for result in results] - latencies: Final = [result[1] for result in results] rejected: Final = [result for result in results if result[0] == 503] - assert set(statuses) <= {200, 503} - assert rejected - assert all(result[2] for result in rejected) - assert _percentile(latencies, 0.99) < 5 - assert liveness_latencies - assert _percentile(liveness_latencies, 0.95) < 0.5 - - duration: Final = max(latencies) - print( - "\nmetric value\n" - f"rps {len(results) / duration:.2f}\n" - f"200 count {statuses.count(200)}\n" - f"503 count {statuses.count(503)}\n" - f"p50 {_percentile(latencies, 0.50):.3f}s\n" - f"p95 {_percentile(latencies, 0.95):.3f}s\n" - f"p99 {_percentile(latencies, 0.99):.3f}s\n" - f"liveness p95 {_percentile(liveness_latencies, 0.95):.3f}s" - ) + assert set(statuses) <= {200, 503}, statuses + assert 1 <= statuses.count(200) <= MAX_IN_FLIGHT + MAX_QUEUED, statuses + assert len(rejected) >= REQUESTS - MAX_IN_FLIGHT - MAX_QUEUED, statuses + assert all(result[1] for result in rejected), rejected finally: proxy_process.terminate() try: diff --git a/tests/integration/configuration/test_basic_proxy_startup.py b/tests/integration/configuration/test_basic_proxy_startup.py new file mode 100644 index 00000000000..79bc6e03338 --- /dev/null +++ b/tests/integration/configuration/test_basic_proxy_startup.py @@ -0,0 +1,58 @@ +""" +This test ensures that the proxy starts and serves requests even with a bad license. + + +in ci/cd config.yml, we set the license to "bad-license" +""" + +import os +from typing import Final + +import aiohttp +import pytest +from tests.integration._support.provider import SharedProvider +from tests.integration._support.wire import Reply + + +PROXY_URL: Final = os.environ.get("INTEGRATION_PROXY_URL", "http://0.0.0.0:4000") +INTEGRATION_RIG: Final = "INTEGRATION_PROXY_URL" in os.environ +MODEL: Final = "openai/gpt-5.4" if INTEGRATION_RIG else "gpt-4" + + +@pytest.mark.asyncio +async def test_health_and_chat_completion(provider: SharedProvider) -> None: + base_url: Final = PROXY_URL.rstrip("/") + async with aiohttp.ClientSession() as session: + async with session.get(f"{base_url}/health/readiness") as response: + assert response.status == 200, await response.text() + readiness_response: Final = await response.json() + assert readiness_response["status"] in ("healthy", "connected") + async with session.get(f"{base_url}/health/liveness") as response: + assert response.status == 200, await response.text() + headers: Final = { + "Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", + "Content-Type": "application/json", + } + if INTEGRATION_RIG: + provider.expect( + Reply( + status=200, + body=b'{"id":"chatcmpl-scripted","object":"chat.completion","created":1700000000,"model":"gpt-5.4","choices":[{"index":0,"message":{"role":"assistant","content":"scripted integration reply"},"finish_reason":"stop"}],"usage":{"prompt_tokens":4,"completion_tokens":5,"total_tokens":9}}', + ) + ) + data: Final = { + "model": MODEL, + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"}, + ], + } + async with session.post(f"{base_url}/chat/completions", headers=headers, json=data) as response: + assert response.status == 200, await response.text() + completion_response: Final = await response.json() + assert "choices" in completion_response + if INTEGRATION_RIG: + requests: Final = provider.received() + assert len(requests) == 1, requests + assert requests[0].method == "POST", requests[0] + assert requests[0].target.endswith("/chat/completions"), requests[0] diff --git a/tests/test_litellm/test_conftest.py b/tests/integration/configuration/test_unit_conftest_isolation.py similarity index 95% rename from tests/test_litellm/test_conftest.py rename to tests/integration/configuration/test_unit_conftest_isolation.py index 6be9e8f5a20..4c8736f0cf9 100644 --- a/tests/test_litellm/test_conftest.py +++ b/tests/integration/configuration/test_unit_conftest_isolation.py @@ -4,7 +4,7 @@ import sys from pathlib import Path from typing import Final -REPO_ROOT: Final = Path(__file__).resolve().parents[2] +REPO_ROOT: Final = Path(__file__).resolve().parents[3] PROXY_BASE_URL_SENSITIVE_NODE: Final = ( "tests/unit/proxy/management_endpoints/test_mcp_management_endpoints.py" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index dab7c1bbd1d..047347ae962 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -4,20 +4,21 @@ import hashlib import json import os import re -from collections.abc import Generator, Iterator, Mapping, Sequence +from collections.abc import AsyncIterator, Generator, Iterator, Mapping, Sequence from contextlib import ExitStack from copy import copy from importlib.metadata import version from pathlib import Path -from typing import Final, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable import httpx import pytest +import pytest_asyncio from execnet import XSpec from pydantic import TypeAdapter from redis import Redis -from tests.integration._support.client import Gateway, eventually, gateway_from_environment +from tests.integration._support.client import Gateway, Scenario, eventually, gateway_from_environment from tests.integration._support.generation import LIFECYCLE_SETTINGS from tests.integration._support.manifest import OWNED_DIRECTORIES from tests.integration._support.ordering import collected_case, fixture_groups, scheduling_groups @@ -26,6 +27,9 @@ from tests.integration._support.routing import RoutingPlugin from tests.integration._support.workers import worker_endpoints, worker_gateway from tests.integration.run import GITHUB_FILES +if TYPE_CHECKING: + from tests.integration._support.actors import ActorSet, GatewayClient, ManagementDatabase, ScratchResources + COLLECTED: Final = pytest.StashKey[tuple[str, ...]]() INVENTORY: Final = pytest.StashKey[tuple[str, ...]]() REPORTS: Final = pytest.StashKey[list[pytest.TestReport]]() @@ -293,6 +297,55 @@ def gateway() -> Iterator[Gateway]: yield value +@pytest.fixture(scope="session") +def session_gateway() -> Iterator[Gateway]: + with gateway_from_environment() as value: + yield value + + +@pytest.fixture +def scenario(gateway: Gateway) -> Iterator[Scenario]: + value: Final[Scenario] + with gateway.scenario() as value: + yield value + + +@pytest_asyncio.fixture(scope="session", loop_scope="session") +async def actors( + session_gateway: Gateway, db: ManagementDatabase +) -> AsyncIterator[ActorSet]: + with session_gateway.scenario() as scenario: + from tests.integration._support.actors import create_actor_set + + value: Final[ActorSet] = await create_actor_set(session_gateway, scenario, db) + yield value + + +@pytest.fixture(scope="session") +def gateway_client(session_gateway: Gateway, actors: ActorSet) -> GatewayClient: + from tests.integration._support.actors import create_gateway_client + + return create_gateway_client(session_gateway, actors) + + +@pytest_asyncio.fixture(scope="session", loop_scope="session") +async def db(session_gateway: Gateway) -> AsyncIterator[ManagementDatabase]: + from tests.integration._support.actors import management_database + + value: Final[ManagementDatabase] + async with management_database(session_gateway) as value: + yield value + + +@pytest_asyncio.fixture(scope="function", loop_scope="session") +async def resources(db: ManagementDatabase) -> AsyncIterator[ScratchResources]: + from tests.integration._support.actors import scratch_resources + + value: Final[ScratchResources] + async with scratch_resources(db) as value: + yield value + + @pytest.fixture(scope="session") def shared_provider_server() -> Iterator[SharedProvider]: if os.environ.get("PYTEST_XDIST_WORKER") and os.environ.get("INTEGRATION_WORKER_ISOLATION") != "1": diff --git a/tests/proxy_behavior/auth/__init__.py b/tests/integration/management/__init__.py similarity index 100% rename from tests/proxy_behavior/auth/__init__.py rename to tests/integration/management/__init__.py diff --git a/tests/proxy_behavior/management/test_credential_migration_endpoint.py b/tests/integration/management/test_credential_migration_endpoint.py similarity index 69% rename from tests/proxy_behavior/management/test_credential_migration_endpoint.py rename to tests/integration/management/test_credential_migration_endpoint.py index b0428195674..795c76370f7 100644 --- a/tests/proxy_behavior/management/test_credential_migration_endpoint.py +++ b/tests/integration/management/test_credential_migration_endpoint.py @@ -13,16 +13,15 @@ the ``user_api_key_auth`` layer (401) before the endpoint's own admin guard runs import pytest -from .conftest import MASTER_KEY pytestmark = pytest.mark.asyncio(loop_scope="session") -async def test_migrate_encryption_check_as_admin_is_read_only(proxy_client): +async def test_migrate_encryption_check_as_admin_is_read_only(gateway_client): """GET /credentials/migrate-encryption/check returns a residual report (no writes).""" - resp = await proxy_client.get( + resp = await gateway_client.get( "/credentials/migrate-encryption/check", - headers={"Authorization": f"Bearer {MASTER_KEY}"}, + headers={"Authorization": f"Bearer {gateway_client.gateway.key}"}, ) assert resp.status_code == 200, resp.text body = resp.json() @@ -30,41 +29,41 @@ async def test_migrate_encryption_check_as_admin_is_read_only(proxy_client): assert "residual_legacy" in body["report"] -async def test_migrate_encryption_check_requires_admin(proxy_client, scratch): +async def test_migrate_encryption_check_requires_admin(gateway_client, resources): """A non-admin key cannot reach the residual scan (auth layer rejects, 401).""" - gen = await proxy_client.post( + gen = await gateway_client.post( "/key/generate", - headers={"Authorization": f"Bearer {MASTER_KEY}"}, - json={"key_alias": scratch.tag("check"), "user_id": scratch.tag("check-user")}, + headers={"Authorization": f"Bearer {gateway_client.gateway.key}"}, + json={"key_alias": resources.tag("check"), "user_id": resources.tag("check-user")}, ) assert gen.status_code == 200, gen.text nonadmin_key = gen.json()["key"] - resp = await proxy_client.get( + resp = await gateway_client.get( "/credentials/migrate-encryption/check", headers={"Authorization": f"Bearer {nonadmin_key}"}, ) assert resp.status_code == 401, resp.text -async def test_migrate_encryption_requires_admin(proxy_client, scratch): +async def test_migrate_encryption_requires_admin(gateway_client, resources): """A non-admin key cannot trigger the migration (auth layer rejects, 401). Rejection happens before any write: the admin-only route check fires in ``user_api_key_auth``, ahead of the endpoint body. """ - gen = await proxy_client.post( + gen = await gateway_client.post( "/key/generate", - headers={"Authorization": f"Bearer {MASTER_KEY}"}, + headers={"Authorization": f"Bearer {gateway_client.gateway.key}"}, json={ - "key_alias": scratch.tag("migrate"), - "user_id": scratch.tag("migrate-user"), + "key_alias": resources.tag("migrate"), + "user_id": resources.tag("migrate-user"), }, ) assert gen.status_code == 200, gen.text nonadmin_key = gen.json()["key"] - resp = await proxy_client.post( + resp = await gateway_client.post( "/credentials/migrate-encryption", headers={"Authorization": f"Bearer {nonadmin_key}"}, ) diff --git a/tests/proxy_behavior/management/test_f7_coverage_closeout.py b/tests/integration/management/test_f7_coverage_closeout.py similarity index 66% rename from tests/proxy_behavior/management/test_f7_coverage_closeout.py rename to tests/integration/management/test_f7_coverage_closeout.py index c78adbbb79d..527b230e543 100644 --- a/tests/proxy_behavior/management/test_f7_coverage_closeout.py +++ b/tests/integration/management/test_f7_coverage_closeout.py @@ -1,3 +1,4 @@ +from typing import Final """Phase 4 F7 — coverage gap-closer scenarios picked from the un-covered ranges left after F1–F6 landed. @@ -27,8 +28,8 @@ from prisma import Json from litellm.proxy.utils import hash_token -from .actors import TEAM_ALPHA, Actor -from .conftest import create_scratch_org, create_scratch_team +from tests.integration._support.actors import TEAM_ALPHA, Actor +from tests.integration._support.actors import create_scratch_org, create_scratch_team pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -43,11 +44,11 @@ pytestmark = pytest.mark.asyncio(loop_scope="session") async def test_org_team_guaranteed_throughput_model_over_bound_rejected( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): org_id = await create_scratch_org( - prisma, - scratch.prefix, + db, + resources.prefix, models=["gpt-4"], metadata={"model_rpm_limit": {"gpt-4": 30}}, ) @@ -55,13 +56,13 @@ async def test_org_team_guaranteed_throughput_model_over_bound_rejected( # forces _check_team_model_specific_limits's `model_specific_rpm_limit` # accumulator to actually accumulate (covers lines 468–478). await create_scratch_team( - prisma, - team_id=scratch.tag("sibling"), + db, + team_id=resources.tag("sibling"), organization_id=org_id, metadata={"model_rpm_limit": {"gpt-4": 20}}, ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - team_id = scratch.tag("new") + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + team_id = resources.tag("new") body = { "team_id": team_id, "team_alias": team_id, @@ -70,7 +71,7 @@ async def test_org_team_guaranteed_throughput_model_over_bound_rejected( "model_rpm_limit": {"gpt-4": 100}, "rpm_limit_type": "guaranteed_throughput", } - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/new", headers={"Authorization": f"Bearer {seeder}"}, json=body, @@ -78,30 +79,30 @@ async def test_org_team_guaranteed_throughput_model_over_bound_rejected( # 20 (sibling) + 100 (new) > 30 (org cap) → guard fires. assert resp.status_code == 400, resp.text assert "RPM" in resp.text, resp.text - rows = await prisma.db.litellm_teamtable.find_many(where={"team_id": team_id}) + rows = await db.db.litellm_teamtable.find_many(where={"team_id": team_id}) assert rows == [] async def test_org_team_guaranteed_throughput_model_tpm_over_bound_rejected( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """Mirror of the rpm scenario but for the model_tpm side — pins _check_team_model_specific_limits's tpm branch (lines 503–521) which the rpm scenario doesn't exercise.""" org_id = await create_scratch_org( - prisma, - scratch.prefix, + db, + resources.prefix, models=["gpt-4"], metadata={"model_tpm_limit": {"gpt-4": 500}}, ) await create_scratch_team( - prisma, - team_id=scratch.tag("sibling"), + db, + team_id=resources.tag("sibling"), organization_id=org_id, metadata={"model_tpm_limit": {"gpt-4": 200}}, ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - team_id = scratch.tag("new") + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + team_id = resources.tag("new") body = { "team_id": team_id, "team_alias": team_id, @@ -110,51 +111,48 @@ async def test_org_team_guaranteed_throughput_model_tpm_over_bound_rejected( "model_tpm_limit": {"gpt-4": 5000}, "tpm_limit_type": "guaranteed_throughput", } - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/new", headers={"Authorization": f"Bearer {seeder}"}, json=body, ) assert resp.status_code == 400, resp.text assert "TPM" in resp.text, resp.text - rows = await prisma.db.litellm_teamtable.find_many(where={"team_id": team_id}) + rows = await db.db.litellm_teamtable.find_many(where={"team_id": team_id}) assert rows == [] -async def test_org_team_guaranteed_throughput_aggregate_runs( - proxy_client, prisma, scratch, world -): - """Aggregate guard's no-op path (line 549-566 — entity_rpm_limit is None - because include_budget_table=False at the call site). The helper still - executes its `allocated_tpm = sum(...)` and `allocated_rpm = sum(...)` - lines, which is the coverage target.""" - org_id = await create_scratch_org(prisma, scratch.prefix, models=["m"]) - await create_scratch_team( - prisma, - team_id=scratch.tag("sibling"), - organization_id=org_id, - tpm_limit=100, - rpm_limit=10, - ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - team_id = scratch.tag("new") - body = { - "team_id": team_id, - "team_alias": team_id, - "organization_id": org_id, - "models": ["m"], - "tpm_limit": 50, - "rpm_limit": 5, - "tpm_limit_type": "guaranteed_throughput", - } - resp = await proxy_client.post( - "/team/new", - headers={"Authorization": f"Bearer {seeder}"}, - json=body, - ) - # No org budget table loaded → check is no-op → 200. - assert resp.status_code == 200, resp.text +@pytest.mark.parametrize("requested_tpm,expected_status", [(50, 400), (20, 200)], ids=["over", "within"]) +async def test_org_team_guaranteed_throughput_counts_sibling_allocations( + requested_tpm: int, expected_status: int, gateway_client, db, resources, actors +): + org_id = await create_scratch_org(db, resources.prefix, models=["m"], tpm_limit=120) + await create_scratch_team( + db, + team_id=resources.tag("sibling"), + organization_id=org_id, + models=["m"], + tpm_limit=100, + ) + team_id = resources.tag("new") + resp = await gateway_client.post( + "/team/new", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={ + "team_id": team_id, + "team_alias": team_id, + "organization_id": org_id, + "models": ["m"], + "tpm_limit": requested_tpm, + "tpm_limit_type": "guaranteed_throughput", + }, + ) + assert resp.status_code == expected_status, resp.text + rows = await db.db.litellm_teamtable.find_many(where={"team_id": team_id}) + assert len(rows) == (1 if expected_status == 200 else 0) + if expected_status == 400: + assert "Allocated TPM limit=100" in resp.text, resp.text # --------------------------------------------------------------------------- # /key/generate — _check_project_key_limits project-not-found branch @@ -164,14 +162,14 @@ async def test_org_team_guaranteed_throughput_aggregate_runs( async def test_key_generate_with_unknown_project_id_rejected( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext body = { - "key_alias": scratch.prefix, - "project_id": f"{scratch.prefix}-ghost-project", + "key_alias": resources.prefix, + "project_id": f"{resources.prefix}-ghost-project", } - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/generate", headers={"Authorization": f"Bearer {seeder}"}, json=body, @@ -180,8 +178,8 @@ async def test_key_generate_with_unknown_project_id_rejected( # The route's exception handler may wrap it; pin both shapes. assert resp.status_code in (400, 404), resp.text assert "project" in resp.text.lower() or "not found" in resp.text.lower(), resp.text - rows = await prisma.db.litellm_verificationtoken.find_many( - where={"key_alias": scratch.prefix} + rows = await db.db.litellm_verificationtoken.find_many( + where={"key_alias": resources.prefix} ) assert rows == [], "rejected key leaked a row" @@ -195,10 +193,10 @@ async def test_key_generate_with_unknown_project_id_rejected( async def _seed_key_for_relocation( - prisma, scratch_prefix: str, *, user_id: str, team_id: str + db, scratch_prefix: str, *, user_id: str, team_id: str ) -> str: cleartext = "sk-" + uuid.uuid4().hex - await prisma.db.litellm_verificationtoken.create( + await db.db.litellm_verificationtoken.create( data={ "token": hash_token(cleartext), "key_alias": f"{scratch_prefix}-key", @@ -212,39 +210,39 @@ async def _seed_key_for_relocation( async def test_team_new_with_team_member_budget_creates_budget_row( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """/team/new with `team_member_budget` routes through TeamMemberBudgetHandler.create_team_member_budget_table (lines 196–248). Observable end-state: a litellm_budgettable row is created and the team's metadata.team_member_budget_id points at it.""" - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - team_id = scratch.tag("team-with-mbudget") - resp = await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + team_id = resources.tag("team-with-mbudget") + resp = await gateway_client.post( "/team/new", headers={"Authorization": f"Bearer {seeder}"}, json={ "team_id": team_id, - "team_alias": scratch.tag("alias"), + "team_alias": resources.tag("alias"), "team_member_budget": 10.0, "team_member_rpm_limit": 100, "team_member_tpm_limit": 1000, }, ) assert resp.status_code == 200, resp.text - team_row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id}) + team_row = await db.db.litellm_teamtable.find_unique(where={"team_id": team_id}) assert team_row is not None budget_id = (team_row.metadata or {}).get("team_member_budget_id") assert ( budget_id is not None ), f"team_member_budget_id not stamped on team metadata: {team_row.metadata!r}" - # The handler writes the per-member budget row under a non-scratch id + # The handler writes the per-member budget row under a non-resources id # pattern (`team--budget-`), so the prefix sweep can't # reclaim it. Cleanup must run even when a downstream assertion fires; # otherwise orphan rows accumulate across CI re-runs. try: - budget_row = await prisma.db.litellm_budgettable.find_unique( + budget_row = await db.db.litellm_budgettable.find_unique( where={"budget_id": budget_id} ) assert budget_row is not None, "team_member_budget row was not created" @@ -252,23 +250,19 @@ async def test_team_new_with_team_member_budget_creates_budget_row( assert budget_row.rpm_limit == 100 assert budget_row.tpm_limit == 1000 finally: - await prisma.db.litellm_teamtable.update( + await db.db.litellm_teamtable.update( where={"team_id": team_id}, data={"metadata": Json({})}, ) - await prisma.db.litellm_budgettable.delete(where={"budget_id": budget_id}) + await db.db.litellm_budgettable.delete(where={"budget_id": budget_id}) async def test_team_update_team_member_budget_upserts( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): - """/team/update with `team_member_budget` against a team that has no - pre-existing team_member_budget_id routes through - TeamMemberBudgetHandler.upsert_team_member_budget_table's else-branch - (lines 294–303), which in turn calls create_team_member_budget_table.""" - team_id = await create_scratch_team(prisma, team_id=scratch.tag("team")) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + team_id = await create_scratch_team(db, team_id=resources.tag("team")) + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/team/update", headers={"Authorization": f"Bearer {seeder}"}, json={ @@ -277,69 +271,70 @@ async def test_team_update_team_member_budget_upserts( }, ) assert resp.status_code == 200, resp.text - team_row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id}) + team_row = await db.db.litellm_teamtable.find_unique(where={"team_id": team_id}) assert team_row is not None budget_id = (team_row.metadata or {}).get("team_member_budget_id") assert budget_id is not None, "team_member_budget_id not upserted" + budget_row = await db.db.litellm_budgettable.find_unique(where={"budget_id": budget_id}) + assert budget_row is not None and budget_row.max_budget == 5.0, budget_row - # See comment on the sibling test — non-prefixed budget row, cleanup - # must survive an assertion failure to avoid orphan accumulation. try: - # (No further assertions today, but the try/finally keeps the - # cleanup contract uniform with the sibling test and is robust to - # future asserts being added here.) - pass + budget_row: Final = await db.db.litellm_budgettable.find_unique( + where={"budget_id": budget_id} + ) + assert budget_row is not None, "team_member_budget row was not created" + assert budget_row.max_budget == 5.0 finally: - await prisma.db.litellm_teamtable.update( + await db.db.litellm_teamtable.update( where={"team_id": team_id}, data={"metadata": Json({})}, ) - await prisma.db.litellm_budgettable.delete(where={"budget_id": budget_id}) + await db.db.litellm_budgettable.delete(where={"budget_id": budget_id}) async def test_key_team_change_accepted_by_target_team_admin( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """Caller is admin of the destination team AND the key's owner — the common_key_access_checks gate requires user_id-match for non-proxy-admin callers, so the team-admin-accepts branch of validate_key_team_change can only be reached when the team admin is also the key holder. Hits validate_key_team_change line 3007–3011.""" - actor_user_id = f"{scratch.prefix}-self-admin" + actor_user_id = f"{resources.prefix}-self-admin" actor_cleartext = "sk-" + uuid.uuid4().hex - await prisma.db.litellm_usertable.create( + await db.db.litellm_usertable.create( data={"user_id": actor_user_id, "user_role": "internal_user"} ) - await prisma.db.litellm_verificationtoken.create( + await db.db.litellm_verificationtoken.create( data={ "token": hash_token(actor_cleartext), - "key_alias": f"{scratch.prefix}-actor-key", - "key_name": f"{scratch.prefix}-actor-key", + "key_alias": f"{resources.prefix}-actor-key", + "key_name": f"{resources.prefix}-actor-key", "user_id": actor_user_id, "models": [], "allowed_routes": ["/key/update"], } ) source_team = await create_scratch_team( - prisma, - team_id=scratch.tag("source"), + db, + team_id=resources.tag("source"), admin_user_ids=[actor_user_id], ) target_team = await create_scratch_team( - prisma, - team_id=scratch.tag("target"), + db, + team_id=resources.tag("target"), admin_user_ids=[actor_user_id], ) key_cleartext = await _seed_key_for_relocation( - prisma, scratch.prefix, user_id=actor_user_id, team_id=source_team + db, resources.prefix, user_id=actor_user_id, team_id=source_team ) - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/update", headers={"Authorization": f"Bearer {actor_cleartext}"}, json={"key": key_cleartext, "team_id": target_team}, ) assert resp.status_code == 200, resp.text - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": hash_token(key_cleartext)} ) assert row is not None diff --git a/tests/proxy_behavior/management/test_f7_key_coverage_push.py b/tests/integration/management/test_f7_key_coverage_push.py similarity index 63% rename from tests/proxy_behavior/management/test_f7_key_coverage_push.py rename to tests/integration/management/test_f7_key_coverage_push.py index 1bf3d1d394e..84b186abd0d 100644 --- a/tests/proxy_behavior/management/test_f7_key_coverage_push.py +++ b/tests/integration/management/test_f7_key_coverage_push.py @@ -1,38 +1,20 @@ -"""Phase 4 F7-extension — additional payload pins pushing -`key_management_endpoints.py` past the 70 % stretch. - -Same pattern as `test_f7_coverage_closeout.py`: each scenario cites the -file:line range it pins and asserts observable end-state, not response-body -snapshots. Targets the largest non-deferred un-covered ranges left after -the first F7 pass: - - * 5942–6063 /key/health logging-metadata path (test_key_logging body) - * 4630–4692 /key/reset_spend happy path + 404 + admin gate - * 4565–4596 _validate_reset_spend_value branches - * 4421–4476 /key/regenerate ghost-key 404 + premium gate - * 6118–6133 _enforce_unique_key_alias duplicate-alias rejection - * 6148–6169 validate_model_max_budget malformed payload rejection - * 4708–4789 validate_key_list_check user/team/org/key_hash branches - -Excluded: `_rotate_master_key` (lines 3997–4123) — deferred per plan §6. -""" - import uuid -from typing import Any, Dict, Optional +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Final, Optional import pytest from prisma import Json from litellm.proxy.utils import hash_token -from .actors import TEAM_ALPHA, TEAM_BETA, Actor -from .conftest import create_scratch_team +from tests.integration._support.actors import TEAM_ALPHA, TEAM_BETA, Actor +from tests.integration._support.actors import create_scratch_key, create_scratch_team pytestmark = pytest.mark.asyncio(loop_scope="session") async def _seed_token( - prisma, + db, scratch_prefix: str, *, suffix: str = "tok", @@ -57,7 +39,7 @@ async def _seed_token( data["metadata"] = Json(metadata) if team_id is not None: data["team_id"] = team_id - await prisma.db.litellm_verificationtoken.create(data=data) + await db.db.litellm_verificationtoken.create(data=data) return cleartext @@ -71,19 +53,19 @@ async def _seed_token( async def test_key_health_with_logging_metadata_runs_test_logging( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): cleartext = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, + db, + resources.prefix, + user_id=actors.keys[Actor.INTERNAL_USER].user_id, metadata={ "logging": [ - {"callback_name": "noop-scratch-callback"}, + {"callback_name": "noop-resources-callback"}, ] }, ) - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/health", headers={"Authorization": f"Bearer {cleartext}"}, ) @@ -92,21 +74,21 @@ async def test_key_health_with_logging_metadata_runs_test_logging( # Either healthy or unhealthy — both pin the logging branch. assert body["key"] in ("healthy", "unhealthy") assert "logging_callbacks" in body - assert body["logging_callbacks"]["callbacks"] == ["noop-scratch-callback"] + assert body["logging_callbacks"]["callbacks"] == ["noop-resources-callback"] async def test_key_health_with_missing_callback_name_rejected( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """test_key_logging raises ValueError if a callback dict lacks callback_name — wrapped by the outer try/except into a 500.""" cleartext = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, + db, + resources.prefix, + user_id=actors.keys[Actor.PROXY_ADMIN].user_id, metadata={"logging": [{"not_callback_name": "x"}]}, ) - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/health", headers={"Authorization": f"Bearer {cleartext}"}, ) @@ -124,25 +106,25 @@ async def test_key_health_with_missing_callback_name_rejected( # --------------------------------------------------------------------------- -async def test_reset_spend_ghost_key_404(proxy_client, scratch, world): - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( - f"/key/sk-{scratch.prefix}-ghost/reset_spend", +async def test_reset_spend_ghost_key_404(gateway_client, resources, actors): + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( + f"/key/sk-{resources.prefix}-ghost/reset_spend", headers={"Authorization": f"Bearer {seeder}"}, json={"reset_to": 0.0}, ) assert resp.status_code == 404, resp.text -async def test_reset_spend_happy_path(proxy_client, prisma, scratch, world): +async def test_reset_spend_happy_path(gateway_client, db, resources, actors): cleartext = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, + db, + resources.prefix, + user_id=actors.keys[Actor.PROXY_ADMIN].user_id, spend=5.0, ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( f"/key/{cleartext}/reset_spend", headers={"Authorization": f"Bearer {seeder}"}, json={"reset_to": 1.0}, @@ -151,7 +133,7 @@ async def test_reset_spend_happy_path(proxy_client, prisma, scratch, world): assert resp.json()["spend"] == 1.0 assert resp.json()["previous_spend"] == 5.0 - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": hash_token(cleartext)} ) assert row is not None @@ -171,16 +153,16 @@ async def test_reset_spend_happy_path(proxy_client, prisma, scratch, world): ids=["negative", "above_current_spend"], ) async def test_reset_spend_validate_value_branches( - reset_to, expected_detail: str, proxy_client, prisma, scratch, world + reset_to, expected_detail: str, gateway_client, db, resources, actors ): cleartext = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, + db, + resources.prefix, + user_id=actors.keys[Actor.PROXY_ADMIN].user_id, spend=5.0, ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( f"/key/{cleartext}/reset_spend", headers={"Authorization": f"Bearer {seeder}"}, json={"reset_to": reset_to}, @@ -188,50 +170,28 @@ async def test_reset_spend_validate_value_branches( assert resp.status_code == 400, resp.text assert expected_detail in resp.text, resp.text # Row spend must be unchanged. - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": hash_token(cleartext)} ) assert row.spend == 5.0, "spend mutated despite validation rejection" -async def test_reset_spend_non_numeric_caught_by_pydantic( - proxy_client, prisma, scratch, world -): - """Pin: non-float reset_to is rejected at the Pydantic layer with 422 - before reaching _validate_reset_spend_value. This documents that the - helper's `isinstance(reset_to, (int, float))` guard at line 4568 is - structurally unreachable via HTTP.""" - cleartext = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, - spend=5.0, - ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( - f"/key/{cleartext}/reset_spend", - headers={"Authorization": f"Bearer {seeder}"}, - json={"reset_to": "not-a-number"}, - ) - assert resp.status_code == 422, resp.text - - async def test_reset_spend_non_admin_caller_rejected( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """_check_proxy_or_team_admin_for_key raises 403 when caller is neither proxy admin nor admin of the key's team.""" - # Seed a key against TEAM_BETA (no actor in our world is admin of beta). + # Seed a key against actors.team_beta_id (no actor in our actors is admin of beta). cleartext = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.CROSS_ORG_USER].user_id, + db, + resources.prefix, + user_id=actors.keys[Actor.CROSS_ORG_USER].user_id, spend=5.0, - team_id=TEAM_BETA, + team_id=actors.team_beta_id, ) - # Caller is a member of TEAM_ALPHA but not admin of TEAM_BETA. - initiator = world.keys[Actor.INTERNAL_USER].cleartext - resp = await proxy_client.post( + # Caller is a member of actors.team_alpha_id but not admin of actors.team_beta_id. + initiator = actors.keys[Actor.INTERNAL_USER].cleartext + resp = await gateway_client.post( f"/key/{cleartext}/reset_spend", headers={"Authorization": f"Bearer {initiator}"}, json={"reset_to": 1.0}, @@ -239,7 +199,7 @@ async def test_reset_spend_non_admin_caller_rejected( # Route-level admin gate may fire first (401) or the helper's own 403 — # both prove the path is guarded; pin either as rejection. assert resp.status_code in (401, 403), resp.text - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": hash_token(cleartext)} ) assert row.spend == 5.0, "spend mutated despite rejection" @@ -252,37 +212,38 @@ async def test_reset_spend_non_admin_caller_rejected( # --------------------------------------------------------------------------- -async def test_regenerate_ghost_key_404(proxy_client, scratch, world): - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( - f"/key/sk-{scratch.prefix}-ghost/regenerate", +async def test_regenerate_ghost_key_404(gateway_client, resources, actors): + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( + f"/key/sk-{resources.prefix}-ghost/regenerate", headers={"Authorization": f"Bearer {seeder}"}, json={}, ) assert resp.status_code == 404, resp.text -async def test_regenerate_no_key_supplied_400(proxy_client, world): - """POST /key/regenerate with no key in path AND no key in body.""" - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( +async def test_regenerate_no_key_supplied_400(gateway_client, actors): + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/key/regenerate", headers={"Authorization": f"Bearer {seeder}"}, json={}, ) assert resp.status_code == 400, resp.text - assert "No key passed in" in resp.text or "key" in resp.text.lower() + error = resp.json()["error"] + assert error["code"] == "400", resp.text + assert "No key passed in." in error["message"], resp.text -async def test_regenerate_happy_path(proxy_client, prisma, scratch, world): +async def test_regenerate_happy_path(gateway_client, db, resources, actors): cleartext = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, + db, + resources.prefix, + user_id=actors.keys[Actor.PROXY_ADMIN].user_id, ) old_hash = hash_token(cleartext) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( f"/key/{cleartext}/regenerate", headers={"Authorization": f"Bearer {seeder}"}, json={}, @@ -294,35 +255,35 @@ async def test_regenerate_happy_path(proxy_client, prisma, scratch, world): assert new_hash != old_hash # Old token should no longer be in active tokens (deleted by regenerate). - old_row = await prisma.db.litellm_verificationtoken.find_unique( + old_row = await db.db.litellm_verificationtoken.find_unique( where={"token": old_hash} ) assert old_row is None, "old token still present after regenerate" # New token should be active. - new_row = await prisma.db.litellm_verificationtoken.find_unique( + new_row = await db.db.litellm_verificationtoken.find_unique( where={"token": new_hash} ) assert new_row is not None, "new token not written after regenerate" -async def test_regenerate_with_explicit_new_key(proxy_client, prisma, scratch, world): +async def test_regenerate_with_explicit_new_key(gateway_client, db, resources, actors): cleartext = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, + db, + resources.prefix, + user_id=actors.keys[Actor.PROXY_ADMIN].user_id, suffix="explicit", ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext new_key = "sk-" + uuid.uuid4().hex - resp = await proxy_client.post( + resp = await gateway_client.post( f"/key/{cleartext}/regenerate", headers={"Authorization": f"Bearer {seeder}"}, json={"new_key": new_key}, ) assert resp.status_code == 200, resp.text assert resp.json()["key"] == new_key - new_row = await prisma.db.litellm_verificationtoken.find_unique( + new_row = await db.db.litellm_verificationtoken.find_unique( where={"token": hash_token(new_key)} ) assert new_row is not None @@ -334,16 +295,16 @@ async def test_regenerate_with_explicit_new_key(proxy_client, prisma, scratch, w # --------------------------------------------------------------------------- -async def test_generate_duplicate_alias_rejected(proxy_client, prisma, scratch, world): - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - alias = scratch.prefix + "-shared-alias" - first = await proxy_client.post( +async def test_generate_duplicate_alias_rejected(gateway_client, db, resources, actors): + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + alias = resources.prefix + "-shared-alias" + first = await gateway_client.post( "/key/generate", headers={"Authorization": f"Bearer {seeder}"}, json={"key_alias": alias}, ) assert first.status_code == 200, first.text - second = await proxy_client.post( + second = await gateway_client.post( "/key/generate", headers={"Authorization": f"Bearer {seeder}"}, json={"key_alias": alias}, @@ -359,14 +320,14 @@ async def test_generate_duplicate_alias_rejected(proxy_client, prisma, scratch, async def test_generate_with_invalid_model_max_budget_rejected( - proxy_client, scratch, world + gateway_client, resources, actors ): - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/key/generate", headers={"Authorization": f"Bearer {seeder}"}, json={ - "key_alias": scratch.prefix, + "key_alias": resources.prefix, # Wrong shape — budget_limit should be numeric; passing a dict # for the model value bypasses the BudgetConfig parse and trips # the validator's exception wrap. @@ -401,14 +362,14 @@ async def test_generate_with_invalid_model_max_budget_rejected( ids=["user_mismatch", "team_mismatch", "org_mismatch"], ) async def test_key_list_non_admin_authz_branches( - filter_kwarg, expected_substring: str, proxy_client, world + filter_kwarg, expected_substring: str, gateway_client, actors ): """Non-admin caller hits validate_key_list_check's three rejection - branches. INTERNAL_USER (Org A, TEAM_ALPHA member) is the caller; each + branches. INTERNAL_USER (Org A, actors.team_alpha_id member) is the caller; each filter targets a foreign user/team/org and trips the matching guard.""" - caller = world.keys[Actor.INTERNAL_USER].cleartext + caller = actors.keys[Actor.INTERNAL_USER].cleartext qs = "&".join(f"{k}={v}" for k, v in filter_kwarg.items()) - resp = await proxy_client.get( + resp = await gateway_client.get( f"/key/list?{qs}", headers={"Authorization": f"Bearer {caller}"}, ) @@ -424,22 +385,22 @@ async def test_key_list_non_admin_authz_branches( async def test_key_bulk_update_mixed_success_and_failure( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext existing = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, + db, + resources.prefix, + user_id=actors.keys[Actor.PROXY_ADMIN].user_id, suffix="bulk1", ) - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/bulk_update", headers={"Authorization": f"Bearer {seeder}"}, json={ "keys": [ {"key": existing, "max_budget": 5.0}, - {"key": "sk-ghost-" + scratch.prefix, "max_budget": 5.0}, + {"key": "sk-ghost-" + resources.prefix, "max_budget": 5.0}, ] }, ) @@ -449,16 +410,16 @@ async def test_key_bulk_update_mixed_success_and_failure( assert len(body["successful_updates"]) == 1 assert len(body["failed_updates"]) == 1 # Re-read confirms the successful key was actually updated. - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": hash_token(existing)} ) assert row.max_budget == 5.0 -async def test_key_bulk_update_non_admin_rejected(proxy_client, world): +async def test_key_bulk_update_non_admin_rejected(gateway_client, actors): """Lines 2631–2635 — admin-only role gate.""" - caller = world.keys[Actor.INTERNAL_USER].cleartext - resp = await proxy_client.post( + caller = actors.keys[Actor.INTERNAL_USER].cleartext + resp = await gateway_client.post( "/key/bulk_update", headers={"Authorization": f"Bearer {caller}"}, json={"keys": [{"key": "sk-x", "max_budget": 1.0}]}, @@ -466,10 +427,10 @@ async def test_key_bulk_update_non_admin_rejected(proxy_client, world): assert resp.status_code in (401, 403), resp.text -async def test_key_bulk_update_empty_keys_rejected(proxy_client, world): +async def test_key_bulk_update_empty_keys_rejected(gateway_client, actors): """Line 2643–2647 — empty keys list rejected.""" - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/key/bulk_update", headers={"Authorization": f"Bearer {seeder}"}, json={"keys": []}, @@ -478,12 +439,12 @@ async def test_key_bulk_update_empty_keys_rejected(proxy_client, world): assert "No keys" in resp.text -async def test_key_bulk_update_exceeds_max_batch_rejected(proxy_client, world): +async def test_key_bulk_update_exceeds_max_batch_rejected(gateway_client, actors): """Lines 2649–2656 — over-batch-size rejection. 501 ghost keys are fine here because validation fires before any update runs.""" - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext over_batch = [{"key": f"sk-{i}", "max_budget": 1.0} for i in range(501)] - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/bulk_update", headers={"Authorization": f"Bearer {seeder}"}, json={"keys": over_batch}, @@ -501,15 +462,15 @@ async def test_key_bulk_update_exceeds_max_batch_rejected(proxy_client, world): async def test_key_update_with_duration_and_budget_duration( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): cleartext = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, + db, + resources.prefix, + user_id=actors.keys[Actor.PROXY_ADMIN].user_id, ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/key/update", headers={"Authorization": f"Bearer {seeder}"}, json={ @@ -519,35 +480,48 @@ async def test_key_update_with_duration_and_budget_duration( }, ) assert resp.status_code == 200, resp.text - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": hash_token(cleartext)} ) - assert row.expires is not None, "expires not stamped from duration" - assert row.budget_reset_at is not None, "budget_reset_at not stamped" - - -async def test_key_update_with_clear_duration(proxy_client, prisma, scratch, world): - """`duration: -1` clears expires (line 1796–1798).""" - cleartext = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, + assert row is not None + assert row.budget_duration == "30d" + now = datetime.now(timezone.utc) + assert timedelta(minutes=55) < row.expires - now <= timedelta(hours=1), row.expires + assert now < row.budget_reset_at <= now + timedelta(days=31), row.budget_reset_at + info = await gateway_client.get( + f"/key/info?key={hash_token(cleartext)}", + headers={"Authorization": f"Bearer {seeder}"}, ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - # First set an expiry - await proxy_client.post( + assert info.status_code == 200, info.text + assert info.json()["info"]["expires"] == row.expires.isoformat() + assert info.json()["info"]["budget_reset_at"] == row.budget_reset_at.isoformat() + + +async def test_key_update_with_clear_duration(gateway_client, db, resources, actors): + cleartext = await _seed_token( + db, + resources.prefix, + user_id=actors.keys[Actor.PROXY_ADMIN].user_id, + ) + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + expiring = await gateway_client.post( "/key/update", headers={"Authorization": f"Bearer {seeder}"}, json={"key": cleartext, "duration": "1h"}, ) - # Then clear it - resp = await proxy_client.post( + assert expiring.status_code == 200, expiring.text + expiring_row = await db.db.litellm_verificationtoken.find_unique( + where={"token": hash_token(cleartext)} + ) + assert expiring_row.expires is not None, "duration=1h did not set expires" + + resp = await gateway_client.post( "/key/update", headers={"Authorization": f"Bearer {seeder}"}, json={"key": cleartext, "duration": "-1"}, ) assert resp.status_code == 200, resp.text - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": hash_token(cleartext)} ) assert row.expires is None, "expires not cleared by duration=-1" @@ -564,27 +538,15 @@ async def test_key_update_with_clear_duration(proxy_client, prisma, scratch, wor # --------------------------------------------------------------------------- -async def test_team_key_bulk_update_missing_team_id_rejected(proxy_client, world): - """Pydantic catches missing team_id at the request layer → 422 before - the handler's own `if not data.team_id` guard at line 2803.""" - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( - "/team/key/bulk_update", - headers={"Authorization": f"Bearer {seeder}"}, - json={"update_fields": {"max_budget": 5.0}}, - ) - assert resp.status_code == 422, resp.text - - -async def test_team_key_bulk_update_no_selector_rejected(proxy_client, world): +async def test_team_key_bulk_update_no_selector_rejected(gateway_client, actors): """Pydantic root-validator catches missing key_ids/all_keys_in_team at the request layer → 422.""" - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/team/key/bulk_update", headers={"Authorization": f"Bearer {seeder}"}, json={ - "team_id": TEAM_ALPHA, + "team_id": actors.team_alpha_id, "update_fields": {"max_budget": 5.0}, }, ) @@ -593,31 +555,31 @@ async def test_team_key_bulk_update_no_selector_rejected(proxy_client, world): async def test_team_key_bulk_update_all_keys_in_team( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """Pin the all_keys_in_team branch (lines 2818–2841) plus the - per-key success path. Seed two scratch keys against the scratch team.""" - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + per-key success path. Seed two resources keys against the resources team.""" + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext team_id = await create_scratch_team( - prisma, - team_id=scratch.tag("team"), - admin_user_ids=[world.keys[Actor.PROXY_ADMIN].user_id], + db, + team_id=resources.tag("team"), + admin_user_ids=[actors.keys[Actor.PROXY_ADMIN].user_id], ) k1 = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, + db, + resources.prefix, + user_id=actors.keys[Actor.PROXY_ADMIN].user_id, suffix="t1", team_id=team_id, ) k2 = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, + db, + resources.prefix, + user_id=actors.keys[Actor.PROXY_ADMIN].user_id, suffix="t2", team_id=team_id, ) - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/key/bulk_update", headers={"Authorization": f"Bearer {seeder}"}, json={ @@ -631,36 +593,36 @@ async def test_team_key_bulk_update_all_keys_in_team( assert body["total_requested"] == 2 # Re-read both keys; max_budget should be set. for k in (k1, k2): - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": hash_token(k)} ) assert row.max_budget == 7.0 async def test_team_key_bulk_update_explicit_key_ids_mixed( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """Pin the explicit-key_ids dedupe + per-key 404 path (lines 2842–2944). Send a real key + a ghost key under the same team_id.""" - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext team_id = await create_scratch_team( - prisma, - team_id=scratch.tag("team"), - admin_user_ids=[world.keys[Actor.PROXY_ADMIN].user_id], + db, + team_id=resources.tag("team"), + admin_user_ids=[actors.keys[Actor.PROXY_ADMIN].user_id], ) real_key = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, + db, + resources.prefix, + user_id=actors.keys[Actor.PROXY_ADMIN].user_id, suffix="real", team_id=team_id, ) - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/key/bulk_update", headers={"Authorization": f"Bearer {seeder}"}, json={ "team_id": team_id, - "key_ids": [real_key, real_key, "sk-ghost-" + scratch.prefix], + "key_ids": [real_key, real_key, "sk-ghost-" + resources.prefix], "update_fields": {"max_budget": 9.0}, }, ) @@ -686,24 +648,24 @@ async def test_team_key_bulk_update_explicit_key_ids_mixed( # --------------------------------------------------------------------------- -async def test_key_info_ghost_key_404(proxy_client, scratch, world): - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.get( - f"/key/info?key=sk-{scratch.prefix}-ghost", +async def test_key_info_ghost_key_404(gateway_client, resources, actors): + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.get( + f"/key/info?key=sk-{resources.prefix}-ghost", headers={"Authorization": f"Bearer {seeder}"}, ) assert resp.status_code == 404, resp.text -async def test_key_info_explicit_existing_key(proxy_client, prisma, scratch, world): +async def test_key_info_explicit_existing_key(gateway_client, db, resources, actors): cleartext = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, + db, + resources.prefix, + user_id=actors.keys[Actor.PROXY_ADMIN].user_id, suffix="info-test", ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.get( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.get( f"/key/info?key={cleartext}", headers={"Authorization": f"Bearer {seeder}"}, ) @@ -715,15 +677,18 @@ async def test_key_info_explicit_existing_key(proxy_client, prisma, scratch, wor assert "token" not in body["info"] -async def test_key_info_no_key_uses_auth_header(proxy_client, world): - """Pin line 3260: `key = key or user_api_key_dict.api_key` — caller's - own key info is returned when no `?key=` is supplied.""" - caller = world.keys[Actor.INTERNAL_USER].cleartext - resp = await proxy_client.get( +async def test_key_info_no_key_uses_auth_header(gateway_client, actors): + caller = actors.keys[Actor.INTERNAL_USER].cleartext + resp = await gateway_client.get( "/key/info", headers={"Authorization": f"Bearer {caller}"}, ) assert resp.status_code == 200, resp.text + body: Final = resp.json() + assert body["key"] == hash_token(caller) + assert body["info"]["user_id"] == actors.keys[Actor.INTERNAL_USER].user_id + assert body["info"]["status"] == "active" + assert "token" not in body["info"] # --------------------------------------------------------------------------- @@ -732,13 +697,13 @@ async def test_key_info_no_key_uses_auth_header(proxy_client, world): # --------------------------------------------------------------------------- -async def test_key_generate_with_budget_limits(proxy_client, prisma, scratch, world): - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( +async def test_key_generate_with_budget_limits(gateway_client, db, resources, actors): + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/key/generate", headers={"Authorization": f"Bearer {seeder}"}, json={ - "key_alias": scratch.prefix, + "key_alias": resources.prefix, "budget_limits": [ {"max_budget": 5.0, "budget_duration": "1d"}, {"max_budget": 20.0, "budget_duration": "30d"}, @@ -746,17 +711,27 @@ async def test_key_generate_with_budget_limits(proxy_client, prisma, scratch, wo }, ) assert resp.status_code == 200, resp.text + cleartext: Final = resp.json()["key"] + info: Final = await gateway_client.get( + f"/key/info?key={hash_token(cleartext)}", + headers={"Authorization": f"Bearer {seeder}"}, + ) + assert info.status_code == 200, info.text + windows: Final = info.json()["info"]["budget_limits"] + assert [ + (window["max_budget"], window["budget_duration"]) for window in windows + ] == [(5.0, "1d"), (20.0, "30d")] -async def test_key_aliases_proxy_admin_unscoped(proxy_client, prisma, scratch, world): - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - alias = scratch.prefix + "-aliases-test" - await proxy_client.post( +async def test_key_aliases_proxy_admin_unscoped(gateway_client, db, resources, actors): + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + alias = resources.prefix + "-aliases-test" + await gateway_client.post( "/key/generate", headers={"Authorization": f"Bearer {seeder}"}, json={"key_alias": alias}, ) - resp = await proxy_client.get( + resp = await gateway_client.get( "/key/aliases?size=10", headers={"Authorization": f"Bearer {seeder}"}, ) @@ -771,36 +746,23 @@ async def test_key_aliases_proxy_admin_unscoped(proxy_client, prisma, scratch, w ), f"newly created alias not in list: {body['aliases']}" -async def test_key_aliases_with_search_filter(proxy_client, prisma, scratch, world): +async def test_key_aliases_with_search_filter(gateway_client, db, resources, actors): """Pin the `search` ILIKE branch (lines 5166–5168).""" - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - unique_alias = scratch.prefix + "-search-unique-tag" - await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + unique_alias = resources.prefix + "-search-unique-tag" + await gateway_client.post( "/key/generate", headers={"Authorization": f"Bearer {seeder}"}, json={"key_alias": unique_alias}, ) - resp = await proxy_client.get( - f"/key/aliases?search={scratch.prefix}-search", + resp = await gateway_client.get( + f"/key/aliases?search={resources.prefix}-search", headers={"Authorization": f"Bearer {seeder}"}, ) assert resp.status_code == 200, resp.text assert unique_alias in resp.json()["aliases"] -async def test_key_aliases_non_admin_scoped(proxy_client, world): - """Non-admin caller routes through _apply_non_admin_alias_scope (line - 5161–5164). The exact alias visibility depends on team membership; the - pin is that the call succeeds with a scoped result.""" - caller = world.keys[Actor.INTERNAL_USER].cleartext - resp = await proxy_client.get( - "/key/aliases?size=10", - headers={"Authorization": f"Bearer {caller}"}, - ) - assert resp.status_code == 200, resp.text - assert "aliases" in resp.json() - - # --------------------------------------------------------------------------- # /key/list with extended filters — pins _build_filter_conditions branches # at lines 5280–5388. Each scenario varies one filter so a different @@ -809,23 +771,51 @@ async def test_key_aliases_non_admin_scoped(proxy_client, world): @pytest.mark.parametrize( - "query", + "filter_name", [ - "include_created_by_keys=true", - "include_team_keys=true", - "return_full_object=true", - "sort_by=created_at&sort_order=asc", - "key_alias=behavior", + "created_by", + "team_keys", + "full_object", + "sort", + "alias_substring", ], ids=["created_by", "team_keys", "full_object", "sort", "alias_substring"], ) -async def test_key_list_filter_branches(query: str, proxy_client, world): - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.get( - f"/key/list?{query}", - headers={"Authorization": f"Bearer {seeder}"}, +async def test_key_list_filter_branches(filter_name: str, gateway_client, resources, actors): + admin: Final = actors.keys[Actor.PROXY_ADMIN] + key_alias: Final = resources.tag("filter-branch") + cleartext: Final = await create_scratch_key( + gateway_client, + admin.cleartext, + resources.prefix, + user_id=admin.user_id, + key_alias=key_alias, + team_id=actors.team_alpha_id, + ) + query: Final = { + "created_by": "include_created_by_keys=true", + "team_keys": "include_team_keys=true", + "full_object": "return_full_object=true", + "sort": "sort_by=created_at&sort_order=asc", + "alias_substring": f"key_alias={key_alias[:-5]}&substring_matching=true", + }[filter_name] + list_query: Final = ( + query + if filter_name == "alias_substring" + else f"{query}&key_alias={key_alias}" + ) + resp: Final = await gateway_client.get( + f"/key/list?{list_query}", + headers={"Authorization": f"Bearer {admin.cleartext}"}, ) assert resp.status_code == 200, resp.text + response_body: Final = resp.json() + entries: Final = response_body["keys"] + assert response_body["total_count"] == 1 + returned_hashes: Final = frozenset( + entry["token"] if isinstance(entry, dict) else entry for entry in entries + ) + assert returned_hashes == {hash_token(cleartext)} # --------------------------------------------------------------------------- @@ -836,17 +826,17 @@ async def test_key_list_filter_branches(query: str, proxy_client, world): async def test_regenerate_with_grace_period_inserts_deprecated_row( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): cleartext = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, + db, + resources.prefix, + user_id=actors.keys[Actor.PROXY_ADMIN].user_id, suffix="grace", ) old_hash = hash_token(cleartext) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( f"/key/{cleartext}/regenerate", headers={"Authorization": f"Bearer {seeder}"}, json={"grace_period": "1h"}, @@ -854,45 +844,45 @@ async def test_regenerate_with_grace_period_inserts_deprecated_row( assert resp.status_code == 200, resp.text new_key = resp.json()["key"] # Old token should now be in the deprecated table. - deprecated_row = await prisma.db.litellm_deprecatedverificationtoken.find_unique( + deprecated_row = await db.db.litellm_deprecatedverificationtoken.find_unique( where={"token": old_hash} ) assert deprecated_row is not None, "old token not retained in deprecated table" assert deprecated_row.active_token_id == hash_token(new_key) assert deprecated_row.revoke_at is not None - # Manual cleanup — scratch prefix sweep doesn't cover this table. - await prisma.db.litellm_deprecatedverificationtoken.delete( + # Manual cleanup — resources prefix sweep doesn't cover this table. + await db.db.litellm_deprecatedverificationtoken.delete( where={"token": old_hash} ) async def test_regenerate_with_invalid_grace_period_format( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """Invalid grace_period format falls through silently (line 4170–4175); regenerate still succeeds but no deprecated row is inserted.""" cleartext = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.PROXY_ADMIN].user_id, + db, + resources.prefix, + user_id=actors.keys[Actor.PROXY_ADMIN].user_id, suffix="grace-bad", ) old_hash = hash_token(cleartext) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( f"/key/{cleartext}/regenerate", headers={"Authorization": f"Bearer {seeder}"}, json={"grace_period": "totally-not-a-duration"}, ) assert resp.status_code == 200, resp.text - deprecated_row = await prisma.db.litellm_deprecatedverificationtoken.find_unique( + deprecated_row = await db.db.litellm_deprecatedverificationtoken.find_unique( where={"token": old_hash} ) assert deprecated_row is None, "deprecated row created despite invalid grace_period" async def test_key_list_key_hash_filter_unauthorized( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """validate_key_list_check's key_hash branch (lines 4766–4789): a cross-tenant non-admin caller asks for a key_hash they don't own → 403. @@ -903,13 +893,13 @@ async def test_key_list_key_hash_filter_unauthorized( the key owner, team member, nor admin — i.e. CROSS_ORG_USER. """ cleartext = await _seed_token( - prisma, - scratch.prefix, - user_id=world.keys[Actor.OWNER].user_id, - team_id=TEAM_ALPHA, + db, + resources.prefix, + user_id=actors.keys[Actor.OWNER].user_id, + team_id=actors.team_alpha_id, ) - caller = world.keys[Actor.CROSS_ORG_USER].cleartext - resp = await proxy_client.get( + caller = actors.keys[Actor.CROSS_ORG_USER].cleartext + resp = await gateway_client.get( f"/key/list?key_hash={hash_token(cleartext)}", headers={"Authorization": f"Bearer {caller}"}, ) diff --git a/tests/proxy_behavior/management/test_key_aliases.py b/tests/integration/management/test_key_aliases.py similarity index 70% rename from tests/proxy_behavior/management/test_key_aliases.py rename to tests/integration/management/test_key_aliases.py index 38ce5cdfaf3..cab06c33f3c 100644 --- a/tests/proxy_behavior/management/test_key_aliases.py +++ b/tests/integration/management/test_key_aliases.py @@ -5,7 +5,7 @@ import pytest from litellm.proxy.utils import hash_token -from .actors import TEAM_ALPHA, TEAM_BETA, Actor +from tests.integration._support.actors import TEAM_ALPHA, TEAM_BETA, Actor pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -16,11 +16,11 @@ pytestmark = pytest.mark.asyncio(loop_scope="session") # own — owned by INTERNAL_USER, no team -> user_id scope only # alpha — owned by OWNER, team TEAM_ALPHA -> team scope for alpha members # beta — owned by CROSS_ORG_USER, TEAM_BETA -async def _seed_alias_keys(prisma, prefix: str, world) -> dict: +async def _seed_alias_keys(db, prefix: str, actors) -> dict: spec = { "own": (Actor.INTERNAL_USER, None), - "alpha": (Actor.OWNER, TEAM_ALPHA), - "beta": (Actor.CROSS_ORG_USER, TEAM_BETA), + "alpha": (Actor.OWNER, actors.team_alpha_id), + "beta": (Actor.CROSS_ORG_USER, actors.team_beta_id), } out = {} for tag, (owner, team_id) in spec.items(): @@ -29,18 +29,18 @@ async def _seed_alias_keys(prisma, prefix: str, world) -> dict: "token": hash_token("sk-" + uuid.uuid4().hex), "key_name": f"{prefix}-{tag}-key", "key_alias": alias, - "user_id": world.keys[owner].user_id, + "user_id": actors.keys[owner].user_id, "models": [], } if team_id is not None: data["team_id"] = team_id - await prisma.db.litellm_verificationtoken.create(data=data) + await db.db.litellm_verificationtoken.create(data=data) out[tag] = alias return out -async def _fetch_aliases(proxy_client, caller_cleartext: str, query: str) -> set: - resp = await proxy_client.get( +async def _fetch_aliases(gateway_client, caller_cleartext: str, query: str) -> set: + resp = await gateway_client.get( f"/key/aliases?{query}&size=100", headers={"Authorization": f"Bearer {caller_cleartext}"}, ) @@ -73,17 +73,17 @@ async def test_key_aliases_visibility( actor: Actor, expected_status: int, expected_tags: FrozenSet[str], - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - aliases = await _seed_alias_keys(prisma, scratch.prefix, world) + aliases = await _seed_alias_keys(db, resources.prefix, actors) known = {v: k for k, v in aliases.items()} - resp = await proxy_client.get( - f"/key/aliases?search={scratch.prefix}&size=100", - headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"}, + resp = await gateway_client.get( + f"/key/aliases?search={resources.prefix}&size=100", + headers={"Authorization": f"Bearer {actors.keys[actor].cleartext}"}, ) assert ( resp.status_code == expected_status @@ -97,23 +97,23 @@ async def test_key_aliases_visibility( ), f"{actor.value}: expected {sorted(expected_tags)}, got {sorted(visible)}" -async def test_key_aliases_team_id_filter(proxy_client, prisma, scratch, world): +async def test_key_aliases_team_id_filter(gateway_client, db, resources, actors): """team_id filter narrows the result to keys of that team.""" - aliases = await _seed_alias_keys(prisma, scratch.prefix, world) + aliases = await _seed_alias_keys(db, resources.prefix, actors) returned = await _fetch_aliases( - proxy_client, - world.keys[Actor.PROXY_ADMIN].cleartext, - f"search={scratch.prefix}&team_id={TEAM_ALPHA}", + gateway_client, + actors.keys[Actor.PROXY_ADMIN].cleartext, + f"search={resources.prefix}&team_id={actors.team_alpha_id}", ) assert returned & set(aliases.values()) == {aliases["alpha"]} -async def test_key_aliases_search_filter(proxy_client, prisma, scratch, world): +async def test_key_aliases_search_filter(gateway_client, db, resources, actors): """search is a case-insensitive substring match on key_alias.""" - aliases = await _seed_alias_keys(prisma, scratch.prefix, world) + aliases = await _seed_alias_keys(db, resources.prefix, actors) returned = await _fetch_aliases( - proxy_client, - world.keys[Actor.PROXY_ADMIN].cleartext, + gateway_client, + actors.keys[Actor.PROXY_ADMIN].cleartext, f"search={aliases['beta']}", ) assert returned & set(aliases.values()) == {aliases["beta"]} diff --git a/tests/proxy_behavior/management/test_key_block_unblock.py b/tests/integration/management/test_key_block_unblock.py similarity index 76% rename from tests/proxy_behavior/management/test_key_block_unblock.py rename to tests/integration/management/test_key_block_unblock.py index 37aa0c0219a..b8d197e4218 100644 --- a/tests/proxy_behavior/management/test_key_block_unblock.py +++ b/tests/integration/management/test_key_block_unblock.py @@ -4,8 +4,8 @@ import pytest from litellm.proxy.utils import hash_token -from .actors import TEAM_ALPHA, TEAM_BETA, Actor -from .conftest import create_scratch_key +from tests.integration._support.actors import TEAM_ALPHA, TEAM_BETA, Actor +from tests.integration._support.actors import create_scratch_key pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -40,26 +40,26 @@ _SCENARIOS = [ ] -async def _seed_target(proxy_client, seeder, scratch_prefix, world, shape, caller): +async def _seed_target(gateway_client, seeder, scratch_prefix, actors, shape, caller): if shape == "self": return await create_scratch_key( - proxy_client, seeder, scratch_prefix, user_id=caller.user_id + gateway_client, seeder, scratch_prefix, user_id=caller.user_id ) if shape == "owner": return await create_scratch_key( - proxy_client, + gateway_client, seeder, scratch_prefix, - user_id=world.keys[Actor.OWNER].user_id, - team_id=TEAM_ALPHA, + user_id=actors.keys[Actor.OWNER].user_id, + team_id=actors.team_alpha_id, ) if shape == "cross_org": return await create_scratch_key( - proxy_client, + gateway_client, seeder, scratch_prefix, - user_id=world.keys[Actor.CROSS_ORG_USER].user_id, - team_id=TEAM_BETA, + user_id=actors.keys[Actor.CROSS_ORG_USER].user_id, + team_id=actors.team_beta_id, ) pytest.fail(f"unknown shape={shape}") # pragma: no cover @@ -75,25 +75,25 @@ async def test_key_block_unblock_authz_matrix( actor: Actor, shape: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - caller = world.keys[actor] - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + caller = actors.keys[actor] + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext target_cleartext = await _seed_target( - proxy_client, seeder, scratch.prefix, world, shape, caller + gateway_client, seeder, resources.prefix, actors, shape, caller ) target_hashed = hash_token(target_cleartext) # /unblock starts from a blocked row so a 200 is observable as True->False. if route == "unblock": - await prisma.db.litellm_verificationtoken.update( + await db.db.litellm_verificationtoken.update( where={"token": target_hashed}, data={"blocked": True} ) - resp = await proxy_client.post( + resp = await gateway_client.post( f"/key/{route}", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={"key": target_cleartext}, @@ -102,7 +102,7 @@ async def test_key_block_unblock_authz_matrix( resp.status_code == expected_status ), f"{route} {actor.value} {shape}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": target_hashed} ) assert row is not None @@ -114,27 +114,27 @@ async def test_key_block_unblock_authz_matrix( assert bool(row.blocked) is (route == "unblock"), "denied but blocked mutated" -async def test_key_block_unblock_round_trip(proxy_client, prisma, scratch, world): +async def test_key_block_unblock_round_trip(gateway_client, db, resources, actors): """PROXY_ADMIN block then unblock flips the blocked column True then False.""" - admin = world.keys[Actor.PROXY_ADMIN] + admin = actors.keys[Actor.PROXY_ADMIN] target = await create_scratch_key( - proxy_client, admin.cleartext, scratch.prefix, user_id=admin.user_id + gateway_client, admin.cleartext, resources.prefix, user_id=admin.user_id ) hashed = hash_token(target) headers = {"Authorization": f"Bearer {admin.cleartext}"} - blocked = await proxy_client.post( + blocked = await gateway_client.post( "/key/block", headers=headers, json={"key": target} ) assert blocked.status_code == 200, blocked.text - row = await prisma.db.litellm_verificationtoken.find_unique(where={"token": hashed}) + row = await db.db.litellm_verificationtoken.find_unique(where={"token": hashed}) assert row is not None and row.blocked is True - unblocked = await proxy_client.post( + unblocked = await gateway_client.post( "/key/unblock", headers=headers, json={"key": target} ) assert unblocked.status_code == 200, unblocked.text - row = await prisma.db.litellm_verificationtoken.find_unique(where={"token": hashed}) + row = await db.db.litellm_verificationtoken.find_unique(where={"token": hashed}) assert row is not None and row.blocked is False @@ -143,13 +143,13 @@ async def test_key_block_unblock_round_trip(proxy_client, prisma, scratch, world "actor", [Actor.PROXY_ADMIN, Actor.TEAM_ADMIN], ids=["proxy_admin", "team_admin"] ) async def test_key_block_unblock_missing_key_returns_404( - route: str, actor: Actor, proxy_client, world + route: str, actor: Actor, gateway_client, actors ): """A well-formed but unseeded key is 404 — not 401/403 — for both the PROXY_ADMIN existence check and the non-admin _check_key_admin_access path.""" - caller = world.keys[actor] + caller = actors.keys[actor] missing = "sk-" + uuid.uuid4().hex - resp = await proxy_client.post( + resp = await gateway_client.post( f"/key/{route}", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={"key": missing}, diff --git a/tests/proxy_behavior/management/test_key_budget_limits.py b/tests/integration/management/test_key_budget_limits.py similarity index 87% rename from tests/proxy_behavior/management/test_key_budget_limits.py rename to tests/integration/management/test_key_budget_limits.py index f5066d60a34..52919d86da7 100644 --- a/tests/proxy_behavior/management/test_key_budget_limits.py +++ b/tests/integration/management/test_key_budget_limits.py @@ -26,8 +26,8 @@ from typing import Any, Dict import pytest -from .actors import Actor -from .conftest import create_scratch_org, create_scratch_team +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_org, create_scratch_team pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -93,24 +93,24 @@ async def test_check_team_key_limits_aggregate( body_extras: Dict[str, Any], expected_status: int, detail_substring, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): team_id = await create_scratch_team( - prisma, - team_id=scratch.tag("team"), + db, + team_id=resources.tag("team"), tpm_limit=team_tpm, rpm_limit=team_rpm, ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext body: Dict[str, Any] = { - "key_alias": scratch.prefix, + "key_alias": resources.prefix, "team_id": team_id, **body_extras, } - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/generate", headers={"Authorization": f"Bearer {seeder}"}, json=body, @@ -121,8 +121,8 @@ async def test_check_team_key_limits_aggregate( if detail_substring is not None: assert detail_substring in resp.text, resp.text - rows = await prisma.db.litellm_verificationtoken.find_many( - where={"key_alias": scratch.prefix} + rows = await db.db.litellm_verificationtoken.find_many( + where={"key_alias": resources.prefix} ) if expected_status == 200: assert len(rows) == 1 @@ -178,23 +178,23 @@ async def test_check_team_key_limits_model_specific( body_extras: Dict[str, Any], expected_status: int, detail_substring, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): team_id = await create_scratch_team( - prisma, - team_id=scratch.tag("team"), + db, + team_id=resources.tag("team"), metadata=team_metadata, ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext body: Dict[str, Any] = { - "key_alias": scratch.prefix, + "key_alias": resources.prefix, "team_id": team_id, **body_extras, } - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/generate", headers={"Authorization": f"Bearer {seeder}"}, json=body, @@ -204,8 +204,8 @@ async def test_check_team_key_limits_model_specific( ), f"{body!r} → {resp.status_code}: {resp.text}" if detail_substring is not None: assert detail_substring in resp.text, resp.text - rows = await prisma.db.litellm_verificationtoken.find_many( - where={"key_alias": scratch.prefix} + rows = await db.db.litellm_verificationtoken.find_many( + where={"key_alias": resources.prefix} ) assert len(rows) == (1 if expected_status == 200 else 0) @@ -273,19 +273,19 @@ async def test_check_org_key_limits_aggregate( body_extras: Dict[str, Any], expected_status: int, detail_substring, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - org_id = await create_scratch_org(prisma, scratch.prefix, **org_budget) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + org_id = await create_scratch_org(db, resources.prefix, **org_budget) + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext body: Dict[str, Any] = { - "key_alias": scratch.prefix, + "key_alias": resources.prefix, "organization_id": org_id, **body_extras, } - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/generate", headers={"Authorization": f"Bearer {seeder}"}, json=body, @@ -295,8 +295,8 @@ async def test_check_org_key_limits_aggregate( ), f"{body!r} → {resp.status_code}: {resp.text}" if detail_substring is not None: assert detail_substring in resp.text, resp.text - rows = await prisma.db.litellm_verificationtoken.find_many( - where={"key_alias": scratch.prefix} + rows = await db.db.litellm_verificationtoken.find_many( + where={"key_alias": resources.prefix} ) assert len(rows) == (1 if expected_status == 200 else 0) @@ -347,19 +347,19 @@ async def test_check_org_key_limits_model_specific( body_extras: Dict[str, Any], expected_status: int, detail_substring, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - org_id = await create_scratch_org(prisma, scratch.prefix, metadata=org_metadata) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + org_id = await create_scratch_org(db, resources.prefix, metadata=org_metadata) + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext body: Dict[str, Any] = { - "key_alias": scratch.prefix, + "key_alias": resources.prefix, "organization_id": org_id, **body_extras, } - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/generate", headers={"Authorization": f"Bearer {seeder}"}, json=body, @@ -369,8 +369,8 @@ async def test_check_org_key_limits_model_specific( ), f"{body!r} → {resp.status_code}: {resp.text}" if detail_substring is not None: assert detail_substring in resp.text, resp.text - rows = await prisma.db.litellm_verificationtoken.find_many( - where={"key_alias": scratch.prefix} + rows = await db.db.litellm_verificationtoken.find_many( + where={"key_alias": resources.prefix} ) assert len(rows) == (1 if expected_status == 200 else 0) diff --git a/tests/proxy_behavior/management/test_key_bulk_update.py b/tests/integration/management/test_key_bulk_update.py similarity index 73% rename from tests/proxy_behavior/management/test_key_bulk_update.py rename to tests/integration/management/test_key_bulk_update.py index 1a57998cece..ba566c19f8a 100644 --- a/tests/proxy_behavior/management/test_key_bulk_update.py +++ b/tests/integration/management/test_key_bulk_update.py @@ -4,8 +4,8 @@ import pytest from litellm.proxy.utils import hash_token -from .actors import Actor -from .conftest import create_scratch_key +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_key pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -37,19 +37,19 @@ _MATRIX = [ async def test_key_bulk_update_authz_matrix( actor: Actor, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - caller = world.keys[actor] - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + caller = actors.keys[actor] + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext target = await create_scratch_key( - proxy_client, seeder, scratch.prefix, user_id=caller.user_id + gateway_client, seeder, resources.prefix, user_id=caller.user_id ) hashed = hash_token(target) - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/bulk_update", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={"keys": [{"key": target, "max_budget": _MARKER_BUDGET}]}, @@ -58,7 +58,7 @@ async def test_key_bulk_update_authz_matrix( resp.status_code == expected_status ), f"{actor.value}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_verificationtoken.find_unique(where={"token": hashed}) + row = await db.db.litellm_verificationtoken.find_unique(where={"token": hashed}) assert row is not None if expected_status == 200: body = resp.json() @@ -69,39 +69,39 @@ async def test_key_bulk_update_authz_matrix( assert row.max_budget != _MARKER_BUDGET, "denied but key mutated" -async def test_key_bulk_update_empty_keys_is_400(proxy_client, world): +async def test_key_bulk_update_empty_keys_is_400(gateway_client, actors): """An empty batch is rejected 400 before any per-key processing.""" - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/bulk_update", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"keys": []}, ) assert resp.status_code == 400, resp.text -async def test_key_bulk_update_over_max_batch_is_400(proxy_client, world): +async def test_key_bulk_update_over_max_batch_is_400(gateway_client, actors): """A batch larger than the 500-key cap is rejected 400.""" items = [{"key": "sk-" + uuid.uuid4().hex} for _ in range(501)] - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/bulk_update", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"keys": items}, ) assert resp.status_code == 400, resp.text async def test_key_bulk_update_per_key_failure_is_isolated( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """One bad key in the batch does not abort the others — it lands in failed_updates while the valid key is still updated.""" - admin = world.keys[Actor.PROXY_ADMIN] + admin = actors.keys[Actor.PROXY_ADMIN] valid = await create_scratch_key( - proxy_client, admin.cleartext, scratch.prefix, user_id=admin.user_id + gateway_client, admin.cleartext, resources.prefix, user_id=admin.user_id ) missing = "sk-" + uuid.uuid4().hex - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/bulk_update", headers={"Authorization": f"Bearer {admin.cleartext}"}, json={ @@ -117,7 +117,7 @@ async def test_key_bulk_update_per_key_failure_is_isolated( assert len(body["successful_updates"]) == 1 assert len(body["failed_updates"]) == 1 - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": hash_token(valid)} ) assert row is not None and row.max_budget == _MARKER_BUDGET diff --git a/tests/proxy_behavior/management/test_key_delete.py b/tests/integration/management/test_key_delete.py similarity index 76% rename from tests/proxy_behavior/management/test_key_delete.py rename to tests/integration/management/test_key_delete.py index 0b483edc056..2920692bd13 100644 --- a/tests/proxy_behavior/management/test_key_delete.py +++ b/tests/integration/management/test_key_delete.py @@ -4,8 +4,8 @@ import pytest from litellm.proxy.utils import hash_token -from .actors import TEAM_ALPHA, TEAM_BETA, Actor -from .conftest import create_scratch_key +from tests.integration._support.actors import TEAM_ALPHA, TEAM_BETA, Actor +from tests.integration._support.actors import create_scratch_key pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -46,40 +46,40 @@ async def test_key_delete_authz_matrix( actor: Actor, target_shape: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - caller = world.keys[actor] - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + caller = actors.keys[actor] + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext if target_shape == "self": target_cleartext = await create_scratch_key( - proxy_client, seeder, scratch.prefix, user_id=caller.user_id + gateway_client, seeder, resources.prefix, user_id=caller.user_id ) elif target_shape == "owner": target_cleartext = await create_scratch_key( - proxy_client, + gateway_client, seeder, - scratch.prefix, - user_id=world.keys[Actor.OWNER].user_id, - team_id=TEAM_ALPHA, + resources.prefix, + user_id=actors.keys[Actor.OWNER].user_id, + team_id=actors.team_alpha_id, ) elif target_shape == "cross_org": target_cleartext = await create_scratch_key( - proxy_client, + gateway_client, seeder, - scratch.prefix, - user_id=world.keys[Actor.CROSS_ORG_USER].user_id, - team_id=TEAM_BETA, + resources.prefix, + user_id=actors.keys[Actor.CROSS_ORG_USER].user_id, + team_id=actors.team_beta_id, ) else: pytest.fail(f"unknown target_shape={target_shape}") target_hashed = hash_token(target_cleartext) - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/delete", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={"keys": [target_cleartext]}, @@ -88,10 +88,10 @@ async def test_key_delete_authz_matrix( resp.status_code == expected_status ), f"{actor.value} {target_shape}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": target_hashed} ) - auth_check = await proxy_client.get( + auth_check = await gateway_client.get( "/key/info", headers={"Authorization": f"Bearer {target_cleartext}"} ) @@ -103,11 +103,11 @@ async def test_key_delete_authz_matrix( assert auth_check.status_code == 200 -async def test_key_delete_missing_key_is_404(proxy_client, world): +async def test_key_delete_missing_key_is_404(gateway_client, actors): """Deleting a key absent from the DB is a 404 — not 401/403.""" - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/delete", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"keys": ["sk-" + uuid.uuid4().hex]}, ) assert resp.status_code == 404, resp.text diff --git a/tests/proxy_behavior/management/test_key_generate.py b/tests/integration/management/test_key_generate.py similarity index 86% rename from tests/proxy_behavior/management/test_key_generate.py rename to tests/integration/management/test_key_generate.py index 851de33d3ff..9d6cd60fe7b 100644 --- a/tests/proxy_behavior/management/test_key_generate.py +++ b/tests/integration/management/test_key_generate.py @@ -2,7 +2,7 @@ from typing import Any, Dict import pytest -from .actors import TEAM_ALPHA, TEAM_BETA, Actor +from tests.integration._support.actors import TEAM_ALPHA, TEAM_BETA, Actor pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -42,15 +42,15 @@ async def test_key_generate_authz_matrix( actor: Actor, body_extras: Dict[str, Any], expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - seeded = world.keys[actor] - body: Dict[str, Any] = {"key_alias": scratch.prefix, **body_extras} + seeded = actors.keys[actor] + body: Dict[str, Any] = {"key_alias": resources.prefix, **body_extras} - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/generate", headers={"Authorization": f"Bearer {seeded.cleartext}"}, json=body, @@ -59,8 +59,8 @@ async def test_key_generate_authz_matrix( resp.status_code == expected_status ), f"{actor.value} {body!r} → {resp.status_code}: {resp.text}" - rows = await prisma.db.litellm_verificationtoken.find_many( - where={"key_alias": scratch.prefix} + rows = await db.db.litellm_verificationtoken.find_many( + where={"key_alias": resources.prefix} ) if expected_status == 200: cleartext = resp.json()["key"] diff --git a/tests/proxy_behavior/management/test_key_health.py b/tests/integration/management/test_key_health.py similarity index 56% rename from tests/proxy_behavior/management/test_key_health.py rename to tests/integration/management/test_key_health.py index 62147e7fa13..af0d985d64b 100644 --- a/tests/proxy_behavior/management/test_key_health.py +++ b/tests/integration/management/test_key_health.py @@ -1,17 +1,17 @@ import pytest -from .actors import Actor +from tests.integration._support.actors import Actor pytestmark = pytest.mark.asyncio(loop_scope="session") # POST /key/health has no role gate — it reflects the caller's OWN key logging -# metadata. The world keys carry no "logging" metadata, so every authenticated +# metadata. The actors keys carry no "logging" metadata, so every authenticated # actor gets 200 with key="healthy". This pins auth-required + route coverage. @pytest.mark.parametrize("actor", list(Actor), ids=[a.value for a in Actor]) -async def test_key_health_each_actor_is_healthy(actor: Actor, proxy_client, world): - caller = world.keys[actor] - resp = await proxy_client.post( +async def test_key_health_each_actor_is_healthy(actor: Actor, gateway_client, actors): + caller = actors.keys[actor] + resp = await gateway_client.post( "/key/health", headers={"Authorization": f"Bearer {caller.cleartext}"}, ) @@ -19,6 +19,6 @@ async def test_key_health_each_actor_is_healthy(actor: Actor, proxy_client, worl assert resp.json()["key"] == "healthy" -async def test_key_health_requires_auth(proxy_client): - resp = await proxy_client.post("/key/health") +async def test_key_health_requires_auth(gateway_client): + resp = await gateway_client.post("/key/health", headers={"Authorization": "Bearer invalid"}) assert resp.status_code == 401, resp.text diff --git a/tests/proxy_behavior/management/test_key_info.py b/tests/integration/management/test_key_info.py similarity index 92% rename from tests/proxy_behavior/management/test_key_info.py rename to tests/integration/management/test_key_info.py index ddcef9fd27b..78226dc1510 100644 --- a/tests/proxy_behavior/management/test_key_info.py +++ b/tests/integration/management/test_key_info.py @@ -1,6 +1,6 @@ import pytest -from .actors import Actor +from tests.integration._support.actors import Actor pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -53,12 +53,12 @@ _SCENARIOS = [ ids=[s[0] for s in _SCENARIOS], ) async def test_key_info_authz_matrix( - actor: Actor, target_actor: Actor, expected_status: int, proxy_client, world + actor: Actor, target_actor: Actor, expected_status: int, gateway_client, actors ): - caller = world.keys[actor] - target = world.keys[target_actor] + caller = actors.keys[actor] + target = actors.keys[target_actor] - resp = await proxy_client.get( + resp = await gateway_client.get( f"/key/info?key={target.cleartext}", headers={"Authorization": f"Bearer {caller.cleartext}"}, ) diff --git a/tests/proxy_behavior/management/test_key_info_v2.py b/tests/integration/management/test_key_info_v2.py similarity index 71% rename from tests/proxy_behavior/management/test_key_info_v2.py rename to tests/integration/management/test_key_info_v2.py index b0fb27a19fa..777a1ab0a25 100644 --- a/tests/proxy_behavior/management/test_key_info_v2.py +++ b/tests/integration/management/test_key_info_v2.py @@ -2,16 +2,16 @@ import uuid import pytest -from .actors import Actor +from tests.integration._support.actors import Actor pytestmark = pytest.mark.asyncio(loop_scope="session") # POST /v2/key/info resolves the posted keys, then drops any key the caller # cannot see via _can_user_query_key_info — silently, no 403. A non-admin sees -# a key it owns (user_id match) or a key whose team it belongs to. The world's +# a key it owns (user_id match) or a key whose team it belongs to. The actors's # TEAM_ALPHA members all see each other's keys; CROSS_ORG_USER and the org -# admins see only their own. The request is posted with every world key, and +# admins see only their own. The request is posted with every actors key, and # the returned info set is asserted to equal the visible subset. _ALPHA_KEYS = frozenset( { @@ -40,14 +40,14 @@ _VISIBILITY = { list(_VISIBILITY.items()), ids=[a.value for a in _VISIBILITY], ) -async def test_key_info_v2_visibility(actor, expected_visible, proxy_client, world): - caller = world.keys[actor] - user_id_to_actor = {world.keys[a].user_id: a for a in Actor} +async def test_key_info_v2_visibility(actor, expected_visible, gateway_client, actors): + caller = actors.keys[actor] + user_id_to_actor = {actors.keys[a].user_id: a for a in Actor} - resp = await proxy_client.post( + resp = await gateway_client.post( "/v2/key/info", headers={"Authorization": f"Bearer {caller.cleartext}"}, - json={"keys": [world.keys[a].cleartext for a in Actor]}, + json={"keys": [actors.keys[a].cleartext for a in Actor]}, ) assert resp.status_code == 200, f"{actor.value}: {resp.status_code} {resp.text}" @@ -62,20 +62,20 @@ async def test_key_info_v2_visibility(actor, expected_visible, proxy_client, wor ) -async def test_key_info_v2_no_body_is_422(proxy_client, world): +async def test_key_info_v2_no_body_is_422(gateway_client, actors): """A request with no body is a 422 — the handler has no keys to resolve.""" - resp = await proxy_client.post( + resp = await gateway_client.post( "/v2/key/info", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, ) assert resp.status_code == 422, resp.text -async def test_key_info_v2_unknown_key_returns_empty_info(proxy_client, world): +async def test_key_info_v2_unknown_key_returns_empty_info(gateway_client, actors): """Keys that resolve to no rows yield an empty info list, not an error.""" - resp = await proxy_client.post( + resp = await gateway_client.post( "/v2/key/info", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"keys": ["sk-" + uuid.uuid4().hex]}, ) assert resp.status_code == 200, resp.text diff --git a/tests/proxy_behavior/management/test_key_list.py b/tests/integration/management/test_key_list.py similarity index 63% rename from tests/proxy_behavior/management/test_key_list.py rename to tests/integration/management/test_key_list.py index 0d3f329950c..b70a3782eda 100644 --- a/tests/proxy_behavior/management/test_key_list.py +++ b/tests/integration/management/test_key_list.py @@ -4,8 +4,8 @@ import pytest from litellm.proxy.utils import hash_token -from .actors import TEAM_ALPHA, Actor -from .conftest import create_scratch_key +from tests.integration._support.actors import TEAM_ALPHA, Actor +from tests.integration._support.actors import create_scratch_key pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -24,13 +24,13 @@ _VISIBILITY = { } -async def _all_visible_hashes(proxy_client, caller_cleartext) -> set: +async def _all_visible_hashes(gateway_client, caller_cleartext) -> set: """Walk every /key/list page — size is capped at 100 by the endpoint, so a single request can truncate PROXY_ADMIN's view on a non-fresh DB.""" hashes: set = set() page = 1 while True: - resp = await proxy_client.get( + resp = await gateway_client.get( f"/key/list?page={page}&size=100", headers={"Authorization": f"Bearer {caller_cleartext}"}, ) @@ -51,12 +51,12 @@ async def _all_visible_hashes(proxy_client, caller_cleartext) -> set: ids=[a.value for a in _VISIBILITY], ) async def test_key_list_visibility( - actor: Actor, expected_visible: FrozenSet[Actor], proxy_client, world + actor: Actor, expected_visible: FrozenSet[Actor], gateway_client, actors ): - caller = world.keys[actor] - hashed_to_actor = {world.keys[a].hashed: a for a in Actor} + caller = actors.keys[actor] + hashed_to_actor = {actors.keys[a].hashed: a for a in Actor} - returned_hashes = await _all_visible_hashes(proxy_client, caller.cleartext) + returned_hashes = await _all_visible_hashes(gateway_client, caller.cleartext) visible_seeded = { hashed_to_actor[h] for h in returned_hashes if h in hashed_to_actor } @@ -66,8 +66,8 @@ async def test_key_list_visibility( ) -async def _list_hashes(proxy_client, caller_cleartext: str, query: str) -> set: - resp = await proxy_client.get( +async def _list_hashes(gateway_client, caller_cleartext: str, query: str) -> set: + resp = await gateway_client.get( f"/key/list?{query}&size=100", headers={"Authorization": f"Bearer {caller_cleartext}"}, ) @@ -80,125 +80,125 @@ async def _list_hashes(proxy_client, caller_cleartext: str, query: str) -> set: return hashes -async def test_key_list_admin_key_alias_substring_match(proxy_client, scratch, world): +async def test_key_list_admin_key_alias_substring_match(gateway_client, resources, actors): """A PROXY_ADMIN's key_alias filter is a case-insensitive substring match when substring_matching=true is requested (the dashboard search box); a narrower fragment selects the subset whose alias contains it. Substring matching is opt-in: without the flag the filter is exact (see test_key_list_admin_key_alias_exact_without_substring_flag).""" - admin = world.keys[Actor.PROXY_ADMIN] + admin = actors.keys[Actor.PROXY_ADMIN] a = await create_scratch_key( - proxy_client, + gateway_client, admin.cleartext, - scratch.prefix, + resources.prefix, user_id=admin.user_id, - key_alias=f"{scratch.prefix}-sub-a", + key_alias=f"{resources.prefix}-sub-a", ) b = await create_scratch_key( - proxy_client, + gateway_client, admin.cleartext, - scratch.prefix, + resources.prefix, user_id=admin.user_id, - key_alias=f"{scratch.prefix}-sub-b", + key_alias=f"{resources.prefix}-sub-b", ) seeded = {hash_token(a), hash_token(b)} broad = await _list_hashes( - proxy_client, + gateway_client, admin.cleartext, - f"key_alias={scratch.prefix}-sub&substring_matching=true", + f"key_alias={resources.prefix}-sub&substring_matching=true", ) assert broad & seeded == seeded narrow = await _list_hashes( - proxy_client, + gateway_client, admin.cleartext, - f"key_alias={scratch.prefix}-sub-a&substring_matching=true", + f"key_alias={resources.prefix}-sub-a&substring_matching=true", ) assert narrow & seeded == {hash_token(a)} async def test_key_list_admin_key_alias_exact_without_substring_flag( - proxy_client, scratch, world + gateway_client, resources, actors ): """Regression guard for the prior exact-match contract: without substring_matching, even a PROXY_ADMIN's key_alias filter is exact, so a fragment of a seeded alias does not select it.""" - admin = world.keys[Actor.PROXY_ADMIN] - full_alias = f"{scratch.prefix}-exactflag" + admin = actors.keys[Actor.PROXY_ADMIN] + full_alias = f"{resources.prefix}-exactflag" key = await create_scratch_key( - proxy_client, + gateway_client, admin.cleartext, - scratch.prefix, + resources.prefix, user_id=admin.user_id, key_alias=full_alias, ) key_hash = hash_token(key) - exact = await _list_hashes(proxy_client, admin.cleartext, f"key_alias={full_alias}") + exact = await _list_hashes(gateway_client, admin.cleartext, f"key_alias={full_alias}") assert key_hash in exact fragment = await _list_hashes( - proxy_client, admin.cleartext, f"key_alias={scratch.prefix}-exactfla" + gateway_client, admin.cleartext, f"key_alias={resources.prefix}-exactfla" ) assert key_hash not in fragment async def test_key_list_non_admin_key_alias_is_exact_match( - proxy_client, scratch, world + gateway_client, resources, actors ): """A non-admin's key_alias filter is exact-match only — substring filtering is restricted to admins. The full alias matches; a fragment does not.""" - caller = world.keys[Actor.INTERNAL_USER] - alias = f"{scratch.prefix}-exact" + caller = actors.keys[Actor.INTERNAL_USER] + alias = f"{resources.prefix}-exact" key = await create_scratch_key( - proxy_client, - world.keys[Actor.PROXY_ADMIN].cleartext, - scratch.prefix, + gateway_client, + actors.keys[Actor.PROXY_ADMIN].cleartext, + resources.prefix, user_id=caller.user_id, key_alias=alias, ) key_hash = hash_token(key) - exact = await _list_hashes(proxy_client, caller.cleartext, f"key_alias={alias}") + exact = await _list_hashes(gateway_client, caller.cleartext, f"key_alias={alias}") assert key_hash in exact fragment = await _list_hashes( - proxy_client, caller.cleartext, f"key_alias={scratch.prefix}-exac" + gateway_client, caller.cleartext, f"key_alias={resources.prefix}-exac" ) assert key_hash not in fragment -async def test_key_list_team_id_filter(proxy_client, scratch, world): +async def test_key_list_team_id_filter(gateway_client, resources, actors): """A team_id filter narrows the listing to keys of that team.""" - admin = world.keys[Actor.PROXY_ADMIN] + admin = actors.keys[Actor.PROXY_ADMIN] team_key = await create_scratch_key( - proxy_client, + gateway_client, admin.cleartext, - scratch.prefix, - user_id=world.keys[Actor.OWNER].user_id, - team_id=TEAM_ALPHA, - key_alias=f"{scratch.prefix}-team", + resources.prefix, + user_id=actors.keys[Actor.OWNER].user_id, + team_id=actors.team_alpha_id, + key_alias=f"{resources.prefix}-team", ) no_team_key = await create_scratch_key( - proxy_client, + gateway_client, admin.cleartext, - scratch.prefix, + resources.prefix, user_id=admin.user_id, - key_alias=f"{scratch.prefix}-noteam", + key_alias=f"{resources.prefix}-noteam", ) - hashes = await _list_hashes(proxy_client, admin.cleartext, f"team_id={TEAM_ALPHA}") + hashes = await _list_hashes(gateway_client, admin.cleartext, f"team_id={actors.team_alpha_id}") assert hash_token(team_key) in hashes assert hash_token(no_team_key) not in hashes -async def test_key_list_non_admin_cannot_filter_other_team(proxy_client, world): +async def test_key_list_non_admin_cannot_filter_other_team(gateway_client, actors): """A non-admin filtering by a team it does not belong to is rejected 403.""" - resp = await proxy_client.get( - f"/key/list?team_id={world.team_beta_id}", + resp = await gateway_client.get( + f"/key/list?team_id={actors.team_beta_id}", headers={ - "Authorization": f"Bearer {world.keys[Actor.INTERNAL_USER].cleartext}" + "Authorization": f"Bearer {actors.keys[Actor.INTERNAL_USER].cleartext}" }, ) assert resp.status_code == 403, resp.text diff --git a/tests/proxy_behavior/management/test_key_regenerate.py b/tests/integration/management/test_key_regenerate.py similarity index 60% rename from tests/proxy_behavior/management/test_key_regenerate.py rename to tests/integration/management/test_key_regenerate.py index 724b8b6d65b..3cd81520047 100644 --- a/tests/proxy_behavior/management/test_key_regenerate.py +++ b/tests/integration/management/test_key_regenerate.py @@ -1,12 +1,7 @@ -import litellm import pytest -from litellm.types.proxy.management_endpoints.ui_sso import ( - LiteLLM_UpperboundKeyGenerateParams, -) - -from .actors import TEAM_ALPHA, TEAM_BETA, Actor -from .conftest import create_scratch_key +from tests.integration._support.actors import TEAM_ALPHA, TEAM_BETA, Actor +from tests.integration._support.actors import create_scratch_key, owned_settings_client pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -39,8 +34,8 @@ _SCENARIOS = [ ] -async def _info(proxy_client, cleartext: str): - return await proxy_client.get( +async def _info(gateway_client, cleartext: str): + return await gateway_client.get( "/key/info", headers={"Authorization": f"Bearer {cleartext}"} ) @@ -54,37 +49,37 @@ async def test_key_regenerate_authz_matrix( actor: Actor, target_shape: str, expected_status: int, - proxy_client, - scratch, - world, + gateway_client, + resources, + actors, ): - caller = world.keys[actor] - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + caller = actors.keys[actor] + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext if target_shape == "self": target_cleartext = await create_scratch_key( - proxy_client, seeder, scratch.prefix, user_id=caller.user_id + gateway_client, seeder, resources.prefix, user_id=caller.user_id ) elif target_shape == "owner": target_cleartext = await create_scratch_key( - proxy_client, + gateway_client, seeder, - scratch.prefix, - user_id=world.keys[Actor.OWNER].user_id, - team_id=TEAM_ALPHA, + resources.prefix, + user_id=actors.keys[Actor.OWNER].user_id, + team_id=actors.team_alpha_id, ) elif target_shape == "cross_org": target_cleartext = await create_scratch_key( - proxy_client, + gateway_client, seeder, - scratch.prefix, - user_id=world.keys[Actor.CROSS_ORG_USER].user_id, - team_id=TEAM_BETA, + resources.prefix, + user_id=actors.keys[Actor.CROSS_ORG_USER].user_id, + team_id=actors.team_beta_id, ) else: pytest.fail(f"unknown target_shape={target_shape}") - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/regenerate", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={"key": target_cleartext}, @@ -96,21 +91,21 @@ async def test_key_regenerate_authz_matrix( if expected_status == 200: new_cleartext = resp.json()["key"] assert new_cleartext.startswith("sk-") and new_cleartext != target_cleartext - assert (await _info(proxy_client, target_cleartext)).status_code == 401 - assert (await _info(proxy_client, new_cleartext)).status_code == 200 + assert (await _info(gateway_client, target_cleartext)).status_code == 401 + assert (await _info(gateway_client, new_cleartext)).status_code == 200 else: # Denied: rotation must not have leaked — old cleartext still works. - assert (await _info(proxy_client, target_cleartext)).status_code == 200 + assert (await _info(gateway_client, target_cleartext)).status_code == 200 -async def test_key_path_regenerate_smoke(proxy_client, scratch, world): +async def test_key_path_regenerate_smoke(gateway_client, resources, actors): """Pins that POST /key/{key:path}/regenerate shares the same handler.""" - caller = world.keys[Actor.PROXY_ADMIN] + caller = actors.keys[Actor.PROXY_ADMIN] target_cleartext = await create_scratch_key( - proxy_client, caller.cleartext, scratch.prefix, user_id=caller.user_id + gateway_client, caller.cleartext, resources.prefix, user_id=caller.user_id ) - resp = await proxy_client.post( + resp = await gateway_client.post( f"/key/{target_cleartext}/regenerate", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={}, @@ -118,48 +113,40 @@ async def test_key_path_regenerate_smoke(proxy_client, scratch, world): assert resp.status_code == 200, resp.text new_cleartext = resp.json()["key"] assert new_cleartext.startswith("sk-") and new_cleartext != target_cleartext - assert (await _info(proxy_client, target_cleartext)).status_code == 401 - assert (await _info(proxy_client, new_cleartext)).status_code == 200 + assert (await _info(gateway_client, target_cleartext)).status_code == 401 + assert (await _info(gateway_client, new_cleartext)).status_code == 200 async def test_key_regenerate_enforces_upperbound_key_params( - proxy_client, scratch, world, monkeypatch + gateway_client, resources, actors, tmp_path ): - """Regenerate runs _enforce_upperbound_key_params: a max_budget above - litellm.upperbound_key_generate_params is rejected 400, a value within the - bound is accepted. Pins #26340 (db8ef44323) — regenerate previously - bypassed the upperbound. upperbound_key_generate_params is module-level - litellm.* state, so monkeypatch save/restores it.""" - admin = world.keys[Actor.PROXY_ADMIN] + admin = actors.keys[Actor.PROXY_ADMIN] over_key = await create_scratch_key( - proxy_client, + gateway_client, admin.cleartext, - scratch.prefix, + resources.prefix, user_id=admin.user_id, - key_alias=f"{scratch.prefix}-over", + key_alias=f"{resources.prefix}-over", ) within_key = await create_scratch_key( - proxy_client, + gateway_client, admin.cleartext, - scratch.prefix, + resources.prefix, user_id=admin.user_id, - key_alias=f"{scratch.prefix}-within", - ) - monkeypatch.setattr( - litellm, - "upperbound_key_generate_params", - LiteLLM_UpperboundKeyGenerateParams(max_budget=100.0), + key_alias=f"{resources.prefix}-within", ) headers = {"Authorization": f"Bearer {admin.cleartext}"} - over = await proxy_client.post( - "/key/regenerate", headers=headers, json={"key": over_key, "max_budget": 500.0} - ) + with owned_settings_client( + gateway_client, tmp_path, {"upperbound_key_generate_params": {"max_budget": 100.0}} + ) as client: + over = await client.post( + "/key/regenerate", headers=headers, json={"key": over_key, "max_budget": 500.0} + ) + within = await client.post( + "/key/regenerate", + headers=headers, + json={"key": within_key, "max_budget": 50.0}, + ) assert over.status_code == 400, over.text - - within = await proxy_client.post( - "/key/regenerate", - headers=headers, - json={"key": within_key, "max_budget": 50.0}, - ) assert within.status_code == 200, within.text diff --git a/tests/proxy_behavior/management/test_key_reset_spend.py b/tests/integration/management/test_key_reset_spend.py similarity index 75% rename from tests/proxy_behavior/management/test_key_reset_spend.py rename to tests/integration/management/test_key_reset_spend.py index fb1c266f655..c43ff1720bd 100644 --- a/tests/proxy_behavior/management/test_key_reset_spend.py +++ b/tests/integration/management/test_key_reset_spend.py @@ -4,8 +4,8 @@ import pytest from litellm.proxy.utils import hash_token -from .actors import TEAM_ALPHA, TEAM_BETA, Actor -from .conftest import create_scratch_key +from tests.integration._support.actors import TEAM_ALPHA, TEAM_BETA, Actor +from tests.integration._support.actors import create_scratch_key pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -42,26 +42,26 @@ _SCENARIOS = [ ] -async def _seed_target(proxy_client, seeder, prefix, world, shape, caller) -> str: +async def _seed_target(gateway_client, seeder, prefix, actors, shape, caller) -> str: if shape == "self": return await create_scratch_key( - proxy_client, seeder, prefix, user_id=caller.user_id + gateway_client, seeder, prefix, user_id=caller.user_id ) if shape == "team_alpha": return await create_scratch_key( - proxy_client, + gateway_client, seeder, prefix, - user_id=world.keys[Actor.OWNER].user_id, - team_id=TEAM_ALPHA, + user_id=actors.keys[Actor.OWNER].user_id, + team_id=actors.team_alpha_id, ) if shape == "team_beta": return await create_scratch_key( - proxy_client, + gateway_client, seeder, prefix, - user_id=world.keys[Actor.CROSS_ORG_USER].user_id, - team_id=TEAM_BETA, + user_id=actors.keys[Actor.CROSS_ORG_USER].user_id, + team_id=actors.team_beta_id, ) pytest.fail(f"unknown shape={shape}") # pragma: no cover @@ -75,22 +75,22 @@ async def test_key_reset_spend_authz_matrix( actor: Actor, shape: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - caller = world.keys[actor] - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + caller = actors.keys[actor] + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext target = await _seed_target( - proxy_client, seeder, scratch.prefix, world, shape, caller + gateway_client, seeder, resources.prefix, actors, shape, caller ) hashed = hash_token(target) - await prisma.db.litellm_verificationtoken.update( + await db.db.litellm_verificationtoken.update( where={"token": hashed}, data={"spend": _SEED_SPEND} ) - resp = await proxy_client.post( + resp = await gateway_client.post( f"/key/{target}/reset_spend", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={"reset_to": _RESET_TO}, @@ -99,7 +99,7 @@ async def test_key_reset_spend_authz_matrix( resp.status_code == expected_status ), f"{actor.value} {shape}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_verificationtoken.find_unique(where={"token": hashed}) + row = await db.db.litellm_verificationtoken.find_unique(where={"token": hashed}) assert row is not None if expected_status == 200: assert row.spend == _RESET_TO @@ -110,25 +110,25 @@ async def test_key_reset_spend_authz_matrix( @pytest.mark.parametrize( "actor", [Actor.PROXY_ADMIN, Actor.TEAM_ADMIN], ids=["proxy_admin", "team_admin"] ) -async def test_key_reset_spend_missing_key_is_404(actor: Actor, proxy_client, world): +async def test_key_reset_spend_missing_key_is_404(actor: Actor, gateway_client, actors): """A well-formed but unseeded key is 404 before any spend validation.""" - resp = await proxy_client.post( + resp = await gateway_client.post( f"/key/sk-{uuid.uuid4().hex}/reset_spend", - headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[actor].cleartext}"}, json={"reset_to": 0.0}, ) assert resp.status_code == 404, resp.text async def test_key_reset_spend_above_current_spend_is_400( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """reset_to above the key's current spend is rejected 400.""" - admin = world.keys[Actor.PROXY_ADMIN] + admin = actors.keys[Actor.PROXY_ADMIN] target = await create_scratch_key( - proxy_client, admin.cleartext, scratch.prefix, user_id=admin.user_id + gateway_client, admin.cleartext, resources.prefix, user_id=admin.user_id ) - resp = await proxy_client.post( + resp = await gateway_client.post( f"/key/{target}/reset_spend", headers={"Authorization": f"Bearer {admin.cleartext}"}, json={"reset_to": 1.0}, diff --git a/tests/proxy_behavior/management/test_key_service_account_generate.py b/tests/integration/management/test_key_service_account_generate.py similarity index 80% rename from tests/proxy_behavior/management/test_key_service_account_generate.py rename to tests/integration/management/test_key_service_account_generate.py index 3b5bbe39754..af33ebc1585 100644 --- a/tests/proxy_behavior/management/test_key_service_account_generate.py +++ b/tests/integration/management/test_key_service_account_generate.py @@ -1,6 +1,6 @@ import pytest -from .actors import Actor +from tests.integration._support.actors import Actor pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -45,23 +45,23 @@ async def test_key_service_account_generate_authz_matrix( actor: Actor, team_target: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - caller = world.keys[actor] + caller = actors.keys[actor] team_id = { - "own": world.team_alpha_id, - "cross_org": world.team_beta_id, + "own": actors.team_alpha_id, + "cross_org": actors.team_beta_id, "none": None, }[team_target] - body = {"key_alias": scratch.prefix} + body = {"key_alias": resources.prefix} if team_id is not None: body["team_id"] = team_id - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/service-account/generate", headers={"Authorization": f"Bearer {caller.cleartext}"}, json=body, @@ -70,8 +70,8 @@ async def test_key_service_account_generate_authz_matrix( resp.status_code == expected_status ), f"{actor.value} {team_target}: {resp.status_code} {resp.text}" - rows = await prisma.db.litellm_verificationtoken.find_many( - where={"key_alias": scratch.prefix} + rows = await db.db.litellm_verificationtoken.find_many( + where={"key_alias": resources.prefix} ) if expected_status == 200: assert len(rows) == 1 @@ -83,16 +83,16 @@ async def test_key_service_account_generate_authz_matrix( async def test_key_service_account_generate_unknown_team_is_400( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """A team_id absent from the database is rejected 400.""" - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/service-account/generate", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - json={"key_alias": scratch.prefix, "team_id": scratch.tag("no-such-team")}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"key_alias": resources.prefix, "team_id": resources.tag("no-such-team")}, ) assert resp.status_code == 400, resp.text - rows = await prisma.db.litellm_verificationtoken.find_many( - where={"key_alias": scratch.prefix} + rows = await db.db.litellm_verificationtoken.find_many( + where={"key_alias": resources.prefix} ) assert rows == [] diff --git a/tests/proxy_behavior/management/test_key_team_change.py b/tests/integration/management/test_key_team_change.py similarity index 74% rename from tests/proxy_behavior/management/test_key_team_change.py rename to tests/integration/management/test_key_team_change.py index 3bd4a0af0e2..dc6a17214c9 100644 --- a/tests/proxy_behavior/management/test_key_team_change.py +++ b/tests/integration/management/test_key_team_change.py @@ -21,8 +21,8 @@ from prisma import Json from litellm.proxy.utils import hash_token -from .actors import TEAM_ALPHA, Actor -from .conftest import create_scratch_team +from tests.integration._support.actors import TEAM_ALPHA, Actor +from tests.integration._support.actors import create_scratch_team pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -30,7 +30,7 @@ KEY_MODEL = "phase4-f2-key-model" async def _seed_key_with_limits( - prisma, + db, scratch_prefix: str, *, user_id: str, @@ -39,7 +39,7 @@ async def _seed_key_with_limits( tpm_limit: Optional[int] = None, rpm_limit: Optional[int] = None, ) -> str: - """Raw-seed a scratch key with explicit models / tpm / rpm — /key/generate + """Raw-seed a resources key with explicit models / tpm / rpm — /key/generate can't set these against a non-throughput team without firing F1's guards. Returns the cleartext key for /key/update calls.""" cleartext = "sk-" + uuid.uuid4().hex @@ -55,45 +55,45 @@ async def _seed_key_with_limits( data["tpm_limit"] = tpm_limit if rpm_limit is not None: data["rpm_limit"] = rpm_limit - await prisma.db.litellm_verificationtoken.create(data=data) + await db.db.litellm_verificationtoken.create(data=data) return cleartext # --------------------------------------------------------------------------- -# Accepted — proxy admin moves a key into a scratch team that has the model, +# Accepted — proxy admin moves a key into a resources team that has the model, # accommodates the limits, and has the key owner as a member. # --------------------------------------------------------------------------- -async def test_key_team_change_accepted(proxy_client, prisma, scratch, world): - owner_id = world.keys[Actor.OWNER].user_id +async def test_key_team_change_accepted(gateway_client, db, resources, actors): + owner_id = actors.keys[Actor.OWNER].user_id target_team = await create_scratch_team( - prisma, - team_id=scratch.tag("target"), + db, + team_id=resources.tag("target"), member_user_ids=[owner_id], models=[KEY_MODEL], tpm_limit=10_000, rpm_limit=1_000, ) key_cleartext = await _seed_key_with_limits( - prisma, - scratch.prefix, + db, + resources.prefix, user_id=owner_id, - team_id=TEAM_ALPHA, + team_id=actors.team_alpha_id, models=[KEY_MODEL], tpm_limit=500, rpm_limit=50, ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/update", headers={"Authorization": f"Bearer {seeder}"}, json={"key": key_cleartext, "team_id": target_team}, ) assert resp.status_code == 200, resp.text - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": hash_token(key_cleartext)} ) assert row is not None @@ -115,9 +115,9 @@ async def test_key_team_change_accepted(proxy_client, prisma, scratch, world): ], ) async def test_key_team_change_rejected_guards( - scenario: str, proxy_client, prisma, scratch, world + scenario: str, gateway_client, db, resources, actors ): - owner_id = world.keys[Actor.OWNER].user_id + owner_id = actors.keys[Actor.OWNER].user_id team_kwargs: Dict[str, Any] = { "member_user_ids": [owner_id], @@ -141,20 +141,20 @@ async def test_key_team_change_rejected_guards( team_kwargs["member_user_ids"] = [] target_team = await create_scratch_team( - prisma, - team_id=scratch.tag("target"), + db, + team_id=resources.tag("target"), **team_kwargs, ) key_cleartext = await _seed_key_with_limits( - prisma, - scratch.prefix, + db, + resources.prefix, user_id=owner_id, - team_id=TEAM_ALPHA, + team_id=actors.team_alpha_id, **key_kwargs, ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/update", headers={"Authorization": f"Bearer {seeder}"}, json={"key": key_cleartext, "team_id": target_team}, @@ -168,11 +168,11 @@ async def test_key_team_change_rejected_guards( 403, ), f"{scenario}: expected 400/403, got {resp.status_code}: {resp.text}" - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": hash_token(key_cleartext)} ) assert row is not None - assert row.team_id == TEAM_ALPHA, ( + assert row.team_id == actors.team_alpha_id, ( f"{scenario}: row team_id mutated despite rejection — " f"got {row.team_id!r}" ) @@ -186,34 +186,34 @@ async def test_key_team_change_rejected_guards( async def test_key_team_change_rejected_initiator_not_admin( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): - owner_id = world.keys[Actor.OWNER].user_id + owner_id = actors.keys[Actor.OWNER].user_id target_team = await create_scratch_team( - prisma, - team_id=scratch.tag("target"), + db, + team_id=resources.tag("target"), # Owner is the key's user_id; member of the team to clear membership - # guard. Internal-user role on world.keys[INTERNAL_USER] is also a - # member of TEAM_ALPHA but never an admin → role-gate path fires. - member_user_ids=[owner_id, world.keys[Actor.INTERNAL_USER].user_id], + # guard. Internal-user role on actors.keys[INTERNAL_USER] is also a + # member of actors.team_alpha_id but never an admin → role-gate path fires. + member_user_ids=[owner_id, actors.keys[Actor.INTERNAL_USER].user_id], models=[KEY_MODEL], tpm_limit=10_000, rpm_limit=1_000, ) key_cleartext = await _seed_key_with_limits( - prisma, - scratch.prefix, + db, + resources.prefix, user_id=owner_id, - team_id=TEAM_ALPHA, + team_id=actors.team_alpha_id, models=[KEY_MODEL], tpm_limit=500, rpm_limit=50, ) - # The internal_user actor is the initiator: a TEAM_ALPHA member, but not + # The internal_user actor is the initiator: a actors.team_alpha_id member, but not # an admin anywhere, and has no team_member_permissions for /key/update. - initiator = world.keys[Actor.INTERNAL_USER].cleartext + initiator = actors.keys[Actor.INTERNAL_USER].cleartext - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/update", headers={"Authorization": f"Bearer {initiator}"}, json={"key": key_cleartext, "team_id": target_team}, @@ -222,8 +222,8 @@ async def test_key_team_change_rejected_initiator_not_admin( 401, 403, ), f"expected 401/403, got {resp.status_code}: {resp.text}" - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": hash_token(key_cleartext)} ) assert row is not None - assert row.team_id == TEAM_ALPHA, "row team_id mutated despite rejection" + assert row.team_id == actors.team_alpha_id, "row team_id mutated despite rejection" diff --git a/tests/proxy_behavior/management/test_key_update.py b/tests/integration/management/test_key_update.py similarity index 75% rename from tests/proxy_behavior/management/test_key_update.py rename to tests/integration/management/test_key_update.py index 7b7f6f5558b..830647e749b 100644 --- a/tests/proxy_behavior/management/test_key_update.py +++ b/tests/integration/management/test_key_update.py @@ -4,8 +4,8 @@ import pytest from litellm.proxy.utils import hash_token -from .actors import TEAM_ALPHA, TEAM_BETA, Actor -from .conftest import create_scratch_key +from tests.integration._support.actors import TEAM_ALPHA, TEAM_BETA, Actor +from tests.integration._support.actors import create_scratch_key pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -50,40 +50,40 @@ async def test_key_update_authz_matrix( actor: Actor, target_shape: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - caller = world.keys[actor] - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + caller = actors.keys[actor] + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext if target_shape == "self": target_cleartext = await create_scratch_key( - proxy_client, seeder, scratch.prefix, user_id=caller.user_id + gateway_client, seeder, resources.prefix, user_id=caller.user_id ) elif target_shape == "owner": target_cleartext = await create_scratch_key( - proxy_client, + gateway_client, seeder, - scratch.prefix, - user_id=world.keys[Actor.OWNER].user_id, - team_id=TEAM_ALPHA, + resources.prefix, + user_id=actors.keys[Actor.OWNER].user_id, + team_id=actors.team_alpha_id, ) elif target_shape == "cross_org": target_cleartext = await create_scratch_key( - proxy_client, + gateway_client, seeder, - scratch.prefix, - user_id=world.keys[Actor.CROSS_ORG_USER].user_id, - team_id=TEAM_BETA, + resources.prefix, + user_id=actors.keys[Actor.CROSS_ORG_USER].user_id, + team_id=actors.team_beta_id, ) else: pytest.fail(f"unknown target_shape={target_shape}") target_hashed = hash_token(target_cleartext) - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/update", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={"key": target_cleartext, "models": [MARKER_MODEL]}, @@ -92,7 +92,7 @@ async def test_key_update_authz_matrix( resp.status_code == expected_status ), f"{actor.value} {target_shape}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": target_hashed} ) assert row is not None @@ -102,35 +102,35 @@ async def test_key_update_authz_matrix( assert row.models != [MARKER_MODEL], "denied but row mutated" -async def _seed_shape(proxy_client, seeder, prefix, world, shape, caller) -> str: +async def _seed_shape(gateway_client, seeder, prefix, actors, shape, caller) -> str: if shape == "self": return await create_scratch_key( - proxy_client, seeder, prefix, user_id=caller.user_id + gateway_client, seeder, prefix, user_id=caller.user_id ) if shape == "owner": return await create_scratch_key( - proxy_client, + gateway_client, seeder, prefix, - user_id=world.keys[Actor.OWNER].user_id, - team_id=TEAM_ALPHA, + user_id=actors.keys[Actor.OWNER].user_id, + team_id=actors.team_alpha_id, ) if shape == "cross_org": return await create_scratch_key( - proxy_client, + gateway_client, seeder, prefix, - user_id=world.keys[Actor.CROSS_ORG_USER].user_id, - team_id=TEAM_BETA, + user_id=actors.keys[Actor.CROSS_ORG_USER].user_id, + team_id=actors.team_beta_id, ) pytest.fail(f"unknown shape={shape}") # pragma: no cover -async def test_key_update_missing_key_is_404(proxy_client, world): +async def test_key_update_missing_key_is_404(gateway_client, actors): """An update targeting a key absent from the DB is a 404 — not 401/403.""" - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/update", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"key": "sk-" + uuid.uuid4().hex, "models": [MARKER_MODEL]}, ) assert resp.status_code == 404, resp.text @@ -154,19 +154,19 @@ async def test_key_update_denied_does_not_touch_budget_counters( actor: Actor, target_shape: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - caller = world.keys[actor] - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + caller = actors.keys[actor] + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext target = await _seed_shape( - proxy_client, seeder, scratch.prefix, world, target_shape, caller + gateway_client, seeder, resources.prefix, actors, target_shape, caller ) target_hashed = hash_token(target) - resp = await proxy_client.post( + resp = await gateway_client.post( "/key/update", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={"key": target, "max_budget": 999.0, "tpm_limit": 888, "rpm_limit": 777}, @@ -175,7 +175,7 @@ async def test_key_update_denied_does_not_touch_budget_counters( resp.status_code == expected_status ), f"{actor.value} {target_shape}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": target_hashed} ) assert row is not None diff --git a/tests/integration/management/test_smoke.py b/tests/integration/management/test_smoke.py new file mode 100644 index 00000000000..7848b702bb6 --- /dev/null +++ b/tests/integration/management/test_smoke.py @@ -0,0 +1,22 @@ +import pytest + + +pytestmark = pytest.mark.asyncio(loop_scope="session") + + +async def test_key_generate_lands_in_db(gateway_client, db, resources): + from litellm.proxy.utils import hash_token + + resp = await gateway_client.post( + "/key/generate", + headers={"Authorization": f"Bearer {gateway_client.gateway.key}"}, + json={"key_alias": resources.prefix}, + ) + assert resp.status_code == 200, resp.text + cleartext = resp.json()["key"] + assert cleartext.startswith("sk-") + + hashed = hash_token(cleartext) + row = await db.db.litellm_verificationtoken.find_unique(where={"token": hashed}) + assert row is not None + assert row.token == hashed != cleartext diff --git a/tests/proxy_behavior/management/test_team_available.py b/tests/integration/management/test_team_available.py similarity index 70% rename from tests/proxy_behavior/management/test_team_available.py rename to tests/integration/management/test_team_available.py index 874c8dd4df7..f37d70b71a5 100644 --- a/tests/proxy_behavior/management/test_team_available.py +++ b/tests/integration/management/test_team_available.py @@ -1,21 +1,21 @@ import pytest -from .actors import Actor +from tests.integration._support.actors import Actor pytestmark = pytest.mark.asyncio(loop_scope="session") # GET /team/available lists teams from -# litellm.default_internal_user_params["available_teams"]. The behavior world +# litellm.default_internal_user_params["available_teams"]. The behavior actors # configures no available_teams, so the handler returns [] for every actor # before it even reads the caller — this is the route-coverage + default-path # pin. /team/available is an info route, so every authenticated actor reaches # the handler. @pytest.mark.parametrize("actor", list(Actor), ids=[a.value for a in Actor]) -async def test_team_available_default_is_empty(actor: Actor, proxy_client, world): - resp = await proxy_client.get( +async def test_team_available_default_is_empty(actor: Actor, gateway_client, actors): + resp = await gateway_client.get( "/team/available", - headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[actor].cleartext}"}, ) assert resp.status_code == 200, f"{actor.value}: {resp.status_code} {resp.text}" assert resp.json() == [] diff --git a/tests/proxy_behavior/management/test_team_block_unblock.py b/tests/integration/management/test_team_block_unblock.py similarity index 62% rename from tests/proxy_behavior/management/test_team_block_unblock.py rename to tests/integration/management/test_team_block_unblock.py index f90ee6ee6d8..106c6ccbdbd 100644 --- a/tests/proxy_behavior/management/test_team_block_unblock.py +++ b/tests/integration/management/test_team_block_unblock.py @@ -1,7 +1,7 @@ import pytest -from .actors import Actor -from .conftest import create_scratch_team +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_team pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -28,10 +28,10 @@ _MATRIX = [ ] -async def _seed_target(prisma, world, shape: str, team_id: str) -> str: - """Raw-seed the scratch target team; returns its organization_id.""" - org_id = world.org_a_id if shape == "alpha" else world.org_b_id - await create_scratch_team(prisma, team_id, organization_id=org_id) +async def _seed_target(db, actors, shape: str, team_id: str) -> str: + """Raw-seed the resources target team; returns its organization_id.""" + org_id = actors.org_a_id if shape == "alpha" else actors.org_b_id + await create_scratch_team(db, team_id, organization_id=org_id) return org_id @@ -46,31 +46,31 @@ async def test_team_block_unblock_authz_matrix( actor: Actor, shape: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - org_id = await _seed_target(prisma, world, shape, scratch.prefix) - caller = world.keys[actor] + org_id = await _seed_target(db, actors, shape, resources.prefix) + caller = actors.keys[actor] # /unblock starts from a blocked row so a 200 is observable as True->False. if route == "unblock": - await prisma.db.litellm_teamtable.update( - where={"team_id": scratch.prefix}, data={"blocked": True} + await db.db.litellm_teamtable.update( + where={"team_id": resources.prefix}, data={"blocked": True} ) - resp = await proxy_client.post( + resp = await gateway_client.post( f"/team/{route}", headers={"Authorization": f"Bearer {caller.cleartext}"}, - json={"team_id": scratch.prefix, "organization_id": org_id}, + json={"team_id": resources.prefix, "organization_id": org_id}, ) assert ( resp.status_code == expected_status ), f"{route} {actor.value} {shape}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) assert row is not None if expected_status == 200: @@ -79,36 +79,36 @@ async def test_team_block_unblock_authz_matrix( assert bool(row.blocked) is (route == "unblock"), "denied but blocked mutated" -async def test_team_block_unblock_round_trip(proxy_client, prisma, scratch, world): +async def test_team_block_unblock_round_trip(gateway_client, db, resources, actors): """PROXY_ADMIN block then unblock flips the blocked column True then False.""" - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id) - headers = {"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"} + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id) + headers = {"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"} - blocked = await proxy_client.post( - "/team/block", headers=headers, json={"team_id": scratch.prefix} + blocked = await gateway_client.post( + "/team/block", headers=headers, json={"team_id": resources.prefix} ) assert blocked.status_code == 200, blocked.text - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) assert row is not None and row.blocked is True - unblocked = await proxy_client.post( - "/team/unblock", headers=headers, json={"team_id": scratch.prefix} + unblocked = await gateway_client.post( + "/team/unblock", headers=headers, json={"team_id": resources.prefix} ) assert unblocked.status_code == 200, unblocked.text - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) assert row is not None and row.blocked is False @pytest.mark.parametrize("route", ["block", "unblock"]) -async def test_team_block_unblock_missing_team_is_404(route: str, proxy_client, world): +async def test_team_block_unblock_missing_team_is_404(route: str, gateway_client, actors): """A team_id absent from the DB is 404 — the existence check precedes authz.""" - resp = await proxy_client.post( + resp = await gateway_client.post( f"/team/{route}", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"team_id": "behavior-pin-no-such-team"}, ) assert resp.status_code == 404, resp.text diff --git a/tests/proxy_behavior/management/test_team_budget_limits.py b/tests/integration/management/test_team_budget_limits.py similarity index 77% rename from tests/proxy_behavior/management/test_team_budget_limits.py rename to tests/integration/management/test_team_budget_limits.py index a172f625e91..e886f729d5a 100644 --- a/tests/proxy_behavior/management/test_team_budget_limits.py +++ b/tests/integration/management/test_team_budget_limits.py @@ -26,14 +26,14 @@ import pytest from litellm.proxy.utils import hash_token -from .actors import Actor -from .conftest import MASTER_KEY, create_scratch_org, create_scratch_team +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_org, create_scratch_team pytestmark = pytest.mark.asyncio(loop_scope="session") async def _seed_scratch_actor_with_caps( - prisma, + db, scratch_prefix: str, *, user_role: str = "internal_user", @@ -42,7 +42,7 @@ async def _seed_scratch_actor_with_caps( tpm_limit: Optional[int] = None, rpm_limit: Optional[int] = None, ) -> str: - """Raw-seed a scratch actor + verification token, with the token carrying + """Raw-seed a resources actor + verification token, with the token carrying explicit caps that flow into `user_api_key_dict` at request time. Returns cleartext key. Used by F3 user-team-limit scenarios where the @@ -54,7 +54,7 @@ async def _seed_scratch_actor_with_caps( """ user_id = f"{scratch_prefix}-team-creator" cleartext = "sk-" + uuid.uuid4().hex - await prisma.db.litellm_usertable.create( + await db.db.litellm_usertable.create( data={ "user_id": user_id, "user_role": user_role, @@ -73,7 +73,7 @@ async def _seed_scratch_actor_with_caps( token_data["tpm_limit"] = tpm_limit if rpm_limit is not None: token_data["rpm_limit"] = rpm_limit - await prisma.db.litellm_verificationtoken.create(data=token_data) + await db.db.litellm_verificationtoken.create(data=token_data) return cleartext @@ -112,28 +112,28 @@ async def test_check_org_team_limits_models_subset( org_models, body_extras: Dict[str, Any], expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - org_id = await create_scratch_org(prisma, scratch.prefix, models=org_models) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - team_id = scratch.tag("team") + org_id = await create_scratch_org(db, resources.prefix, models=org_models) + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + team_id = resources.tag("team") body: Dict[str, Any] = { "team_id": team_id, - "team_alias": scratch.prefix, + "team_alias": resources.prefix, "organization_id": org_id, **body_extras, } - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/new", headers={"Authorization": f"Bearer {seeder}"}, json=body, ) assert resp.status_code == expected_status, f"{body!r} → {resp.status_code}: {resp.text}" - rows = await prisma.db.litellm_teamtable.find_many(where={"team_id": team_id}) + rows = await db.db.litellm_teamtable.find_many(where={"team_id": team_id}) assert len(rows) == (1 if expected_status == 200 else 0) @@ -191,26 +191,26 @@ async def test_check_org_team_limits_budget_enforced( org_budget, body_extras: Dict[str, Any], expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - org_id = await create_scratch_org(prisma, scratch.prefix, **org_budget) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - team_id = scratch.tag("team") - resp = await proxy_client.post( + org_id = await create_scratch_org(db, resources.prefix, **org_budget) + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + team_id = resources.tag("team") + resp = await gateway_client.post( "/team/new", headers={"Authorization": f"Bearer {seeder}"}, json={ "team_id": team_id, - "team_alias": scratch.prefix, + "team_alias": resources.prefix, "organization_id": org_id, **body_extras, }, ) assert resp.status_code == expected_status, f"{body_extras!r} → {resp.status_code}: {resp.text}" - rows = await prisma.db.litellm_teamtable.find_many(where={"team_id": team_id}) + rows = await db.db.litellm_teamtable.find_many(where={"team_id": team_id}) assert len(rows) == (1 if expected_status == 200 else 0) @@ -223,21 +223,21 @@ async def test_check_org_team_limits_budget_enforced_on_update( org_budget, body_extras: Dict[str, Any], expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - org_id = await create_scratch_org(prisma, scratch.prefix, **org_budget) - team_id = await create_scratch_team(prisma, scratch.tag("team"), organization_id=org_id) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + org_id = await create_scratch_org(db, resources.prefix, **org_budget) + team_id = await create_scratch_team(db, resources.tag("team"), organization_id=org_id) + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/team/update", headers={"Authorization": f"Bearer {seeder}"}, json={"team_id": team_id, **body_extras}, ) assert resp.status_code == expected_status, f"{body_extras!r} → {resp.status_code}: {resp.text}" - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id}) + row = await db.db.litellm_teamtable.find_unique(where={"team_id": team_id}) assert row is not None persisted = {field: getattr(row, field) for field in body_extras} assert (persisted == body_extras) == (expected_status == 200) @@ -309,18 +309,18 @@ async def test_check_user_team_limits( actor_caps, body_extras: Dict[str, Any], expected_status: int, - proxy_client, - prisma, - scratch, + gateway_client, + db, + resources, ): - caller = await _seed_scratch_actor_with_caps(prisma, scratch.prefix, **actor_caps) - team_id = scratch.tag("team") - resp = await proxy_client.post( + caller = await _seed_scratch_actor_with_caps(db, resources.prefix, **actor_caps) + team_id = resources.tag("team") + resp = await gateway_client.post( "/team/new", headers={"Authorization": f"Bearer {caller}"}, json={ "team_id": team_id, - "team_alias": scratch.prefix, + "team_alias": resources.prefix, # Standalone team — no organization_id, so user-limit guard fires. **body_extras, }, @@ -329,7 +329,7 @@ async def test_check_user_team_limits( f"caps={actor_caps} body={body_extras} → {resp.status_code}: {resp.text}" ) - rows = await prisma.db.litellm_teamtable.find_many(where={"team_id": team_id}) + rows = await db.db.litellm_teamtable.find_many(where={"team_id": team_id}) assert len(rows) == (1 if expected_status == 200 else 0) @@ -351,16 +351,16 @@ async def test_check_user_team_limits( ids=["raise_with_generous_personal_budget", "lower_with_tiny_personal_budget"], ) async def test_team_admin_cannot_change_budget_while_max_budget_is_not_editable( - proxy_client, prisma, scratch, personal_budget: float, requested_budget: float + gateway_client, db, resources, personal_budget: float, requested_budget: float ): - caller_cleartext = await _seed_scratch_actor_with_caps(prisma, scratch.prefix, max_budget=personal_budget) + caller_cleartext = await _seed_scratch_actor_with_caps(db, resources.prefix, max_budget=personal_budget) team_id = await create_scratch_team( - prisma, - team_id=scratch.tag("team"), - admin_user_ids=[f"{scratch.prefix}-team-creator"], + db, + team_id=resources.tag("team"), + admin_user_ids=[f"{resources.prefix}-team-creator"], max_budget=500.0, ) - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/update", headers={"Authorization": f"Bearer {caller_cleartext}"}, json={"team_id": team_id, "max_budget": requested_budget}, @@ -368,69 +368,69 @@ async def test_team_admin_cannot_change_budget_while_max_budget_is_not_editable( assert resp.status_code == 403, resp.text assert "Team admin editable fields" in resp.text, resp.text - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id}) + row = await db.db.litellm_teamtable.find_unique(where={"team_id": team_id}) assert row is not None assert row.max_budget == 500.0, "a refused update must leave the team budget unchanged" -async def test_proxy_admin_raise_budget_allowed(proxy_client, prisma, scratch): +async def test_proxy_admin_raise_budget_allowed(gateway_client, db, resources): """A proxy admin may raise a team's budget.""" team_id = await create_scratch_team( - prisma, - team_id=scratch.tag("team"), - admin_user_ids=[f"{scratch.prefix}-team-creator"], + db, + team_id=resources.tag("team"), + admin_user_ids=[f"{resources.prefix}-team-creator"], max_budget=50.0, ) # MASTER_KEY acts as proxy admin. - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/update", - headers={"Authorization": f"Bearer {MASTER_KEY}"}, + headers={"Authorization": f"Bearer {gateway_client.gateway.key}"}, json={"team_id": team_id, "max_budget": 999.0}, ) assert resp.status_code == 200, resp.text - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id}) + row = await db.db.litellm_teamtable.find_unique(where={"team_id": team_id}) assert row is not None assert row.max_budget == 999.0, "proxy admin should be able to raise the budget" -async def test_team_admin_remove_budget_cap_blocked(proxy_client, prisma, scratch): +async def test_team_admin_remove_budget_cap_blocked(gateway_client, db, resources): """A team admin cannot strip the team's cap (max_budget=null); removing the ceiling is the strongest possible raise -> proxy-admin only.""" - caller_cleartext = await _seed_scratch_actor_with_caps(prisma, scratch.prefix, max_budget=100000.0) + caller_cleartext = await _seed_scratch_actor_with_caps(db, resources.prefix, max_budget=100000.0) team_id = await create_scratch_team( - prisma, - team_id=scratch.tag("team"), - admin_user_ids=[f"{scratch.prefix}-team-creator"], + db, + team_id=resources.tag("team"), + admin_user_ids=[f"{resources.prefix}-team-creator"], max_budget=50.0, ) - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/update", headers={"Authorization": f"Bearer {caller_cleartext}"}, json={"team_id": team_id, "max_budget": None}, ) assert resp.status_code == 403, resp.text - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id}) + row = await db.db.litellm_teamtable.find_unique(where={"team_id": team_id}) assert row is not None assert row.max_budget == 50.0, "team budget cap must not be removed by a team admin" -async def test_proxy_admin_remove_budget_cap_allowed(proxy_client, prisma, scratch): +async def test_proxy_admin_remove_budget_cap_allowed(gateway_client, db, resources): """A proxy admin may remove a team's cap (max_budget=null).""" team_id = await create_scratch_team( - prisma, - team_id=scratch.tag("team"), - admin_user_ids=[f"{scratch.prefix}-team-creator"], + db, + team_id=resources.tag("team"), + admin_user_ids=[f"{resources.prefix}-team-creator"], max_budget=50.0, ) - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/update", - headers={"Authorization": f"Bearer {MASTER_KEY}"}, + headers={"Authorization": f"Bearer {gateway_client.gateway.key}"}, json={"team_id": team_id, "max_budget": None}, ) assert resp.status_code == 200, resp.text - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id}) + row = await db.db.litellm_teamtable.find_unique(where={"team_id": team_id}) assert row is not None assert row.max_budget is None, "proxy admin should be able to remove the cap" diff --git a/tests/integration/management/test_team_bulk_member_add.py b/tests/integration/management/test_team_bulk_member_add.py new file mode 100644 index 00000000000..90bb9670cf2 --- /dev/null +++ b/tests/integration/management/test_team_bulk_member_add.py @@ -0,0 +1,105 @@ +import pytest + +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_team + +pytestmark = pytest.mark.asyncio(loop_scope="session") + + +def _member_ids(row) -> list: + return [m["user_id"] for m in (row.members_with_roles or [])] + + +async def test_team_bulk_member_add_proxy_admin_adds_explicit_members( + gateway_client, db, resources, actors +): + """PROXY_ADMIN bulk-adds an explicit member list to a resources team.""" + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id) + new_member = resources.tag("m1") + resp = await gateway_client.post( + "/team/bulk_member_add", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={ + "team_id": resources.prefix, + "members": [{"user_id": new_member, "role": "user"}], + }, + ) + assert resp.status_code == 200, resp.text + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} + ) + assert row is not None and new_member in _member_ids(row) + + +async def test_team_bulk_member_add_empty_members_is_400( + gateway_client, db, resources, actors +): + """An empty member list (with all_users unset) is rejected 400.""" + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id) + resp = await gateway_client.post( + "/team/bulk_member_add", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"team_id": resources.prefix, "members": []}, + ) + assert resp.status_code == 400, resp.text + + +async def test_team_bulk_member_add_over_max_batch_is_400( + gateway_client, db, resources, actors +): + """A member list larger than the 500-member cap is rejected 400.""" + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id) + members = [ + {"user_id": f"{resources.prefix}-u{i}", "role": "user"} for i in range(501) + ] + resp = await gateway_client.post( + "/team/bulk_member_add", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"team_id": resources.prefix, "members": members}, + ) + assert resp.status_code == 400, resp.text + + +@pytest.mark.parametrize( + "actor", + [Actor.TEAM_ADMIN, Actor.INTERNAL_USER], + ids=["team_admin", "internal_user"], +) +async def test_team_bulk_member_add_non_admin_is_401( + actor: Actor, gateway_client, db, resources, actors +): + """/team/bulk_member_add is neither an internal_user nor a self-managed + route — a non-proxy-admin with no org context is 401 at the route gate.""" + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id) + resp = await gateway_client.post( + "/team/bulk_member_add", + headers={"Authorization": f"Bearer {actors.keys[actor].cleartext}"}, + json={ + "team_id": resources.prefix, + "members": [{"user_id": resources.tag("m"), "role": "user"}], + }, + ) + assert resp.status_code == 401, f"{actor.value}: {resp.status_code} {resp.text}" + + +async def test_team_bulk_member_add_all_users_proxy_admin( + gateway_client, db, resources, actors +): + """all_users=True pulls every user in the DB into the team. The route is + reachable only by PROXY_ADMIN (the route gate 401s every other actor — even + an org admin with organization_id in the body), so the handler's own + all_users PROXY_ADMIN gate is never the deciding check at the boundary.""" + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id) + resp = await gateway_client.post( + "/team/bulk_member_add", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"team_id": resources.prefix, "all_users": True}, + ) + assert resp.status_code == 200, resp.text + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} + ) + assert row is not None + member_ids = _member_ids(row) + # every actors actor is a user in the DB, so all are now team members + assert actors.keys[Actor.INTERNAL_USER].user_id in member_ids diff --git a/tests/proxy_behavior/management/test_team_bulk_member_delete.py b/tests/integration/management/test_team_bulk_member_delete.py similarity index 54% rename from tests/proxy_behavior/management/test_team_bulk_member_delete.py rename to tests/integration/management/test_team_bulk_member_delete.py index 0818fb25dfe..49108a7cf4e 100644 --- a/tests/proxy_behavior/management/test_team_bulk_member_delete.py +++ b/tests/integration/management/test_team_bulk_member_delete.py @@ -1,8 +1,8 @@ import pytest from prisma import Json -from .actors import Actor -from .conftest import create_scratch_team, create_scratch_user +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_team, create_scratch_user pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -28,20 +28,20 @@ _MATRIX = [ ] -async def _seed_target(prisma, world, shape: str, team_id: str, victim_ids: list) -> None: +async def _seed_target(db, actors, shape: str, team_id: str, victim_ids: list) -> None: if shape == "alpha": await create_scratch_team( - prisma, + db, team_id, - organization_id=world.org_a_id, - admin_user_ids=[world.keys[Actor.TEAM_ADMIN].user_id], + organization_id=actors.org_a_id, + admin_user_ids=[actors.keys[Actor.TEAM_ADMIN].user_id], member_user_ids=victim_ids, ) elif shape == "beta": await create_scratch_team( - prisma, + db, team_id, - organization_id=world.org_b_id, + organization_id=actors.org_b_id, member_user_ids=victim_ids, ) else: # pragma: no cover - guard @@ -61,18 +61,18 @@ async def test_team_bulk_member_delete_authz_matrix( actor: Actor, shape: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - victims = [scratch.tag("v1"), scratch.tag("v2")] - keep = scratch.tag("keep") - await _seed_target(prisma, world, shape, scratch.prefix, victims + [keep]) - caller = world.keys[actor] + victims = [resources.tag("v1"), resources.tag("v2")] + keep = resources.tag("keep") + await _seed_target(db, actors, shape, resources.prefix, victims + [keep]) + caller = actors.keys[actor] - resp = await proxy_client.post( - f"/management/v1/teams/{scratch.prefix}/members/bulk_delete", + resp = await gateway_client.post( + f"/management/v1/teams/{resources.prefix}/members/bulk_delete", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={"members": [{"user_id": v} for v in victims]}, ) @@ -81,7 +81,7 @@ async def test_team_bulk_member_delete_authz_matrix( assert resp.headers["content-type"] == "application/problem+json" assert resp.json()["type"] == "urn:litellm:error:forbidden" - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + row = await db.db.litellm_teamtable.find_unique(where={"team_id": resources.prefix}) assert row is not None assert keep in _member_ids(row), "unrelated member removed" if expected_status == 200: @@ -91,15 +91,15 @@ async def test_team_bulk_member_delete_authz_matrix( assert set(victims) <= set(_member_ids(row)), "denied but members removed" -async def test_team_bulk_member_delete_reports_each_row_in_order(proxy_client, prisma, scratch, world): - victim = scratch.tag("victim") - keep = scratch.tag("keep") - stranger = scratch.tag("stranger") - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id, member_user_ids=[victim, keep]) +async def test_team_bulk_member_delete_reports_each_row_in_order(gateway_client, db, resources, actors): + victim = resources.tag("victim") + keep = resources.tag("keep") + stranger = resources.tag("stranger") + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id, member_user_ids=[victim, keep]) - resp = await proxy_client.post( - f"/management/v1/teams/{scratch.prefix}/members/bulk_delete", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + resp = await gateway_client.post( + f"/management/v1/teams/{resources.prefix}/members/bulk_delete", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"members": [{"user_id": stranger}, {"user_id": victim}]}, ) assert resp.status_code == 200, resp.text @@ -111,47 +111,47 @@ async def test_team_bulk_member_delete_reports_each_row_in_order(proxy_client, p ] assert body["data"][0]["error"] == "User not found in team" - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) - assert row is not None and _member_ids(row) == [keep] + row = await db.db.litellm_teamtable.find_unique(where={"team_id": resources.prefix}) + assert row is not None and set(_member_ids(row)) == {keep, "default_user_id"} async def test_team_bulk_member_delete_by_id_removes_a_legacy_email_only_roster_entry( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): - email = f"{scratch.prefix}@example.com" - victim = await create_scratch_user(prisma, scratch.prefix, suffix="victim", user_email=email) - keep = scratch.tag("keep") - await prisma.db.litellm_teamtable.create( + email = f"{resources.prefix}@example.com" + victim = await create_scratch_user(db, resources.prefix, suffix="victim", user_email=email) + keep = resources.tag("keep") + await db.db.litellm_teamtable.create( data={ - "team_id": scratch.prefix, - "team_alias": scratch.prefix, - "organization_id": world.org_a_id, + "team_id": resources.prefix, + "team_alias": resources.prefix, + "organization_id": actors.org_a_id, "members_with_roles": Json([{"user_email": email, "role": "user"}, {"user_id": keep, "role": "user"}]), } ) - await prisma.db.litellm_usertable.update(where={"user_id": victim}, data={"teams": [scratch.prefix]}) + await db.db.litellm_usertable.update(where={"user_id": victim}, data={"teams": [resources.prefix]}) - resp = await proxy_client.post( - f"/management/v1/teams/{scratch.prefix}/members/bulk_delete", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + resp = await gateway_client.post( + f"/management/v1/teams/{resources.prefix}/members/bulk_delete", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"members": [{"user_id": victim}]}, ) assert resp.status_code == 200, resp.text assert [(r["user_id"], r["success"]) for r in resp.json()["data"]] == [(victim, True)] - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + row = await db.db.litellm_teamtable.find_unique(where={"team_id": resources.prefix}) assert row is not None and [(m["user_id"], m.get("user_email")) for m in row.members_with_roles] == [(keep, None)] - user = await prisma.db.litellm_usertable.find_unique(where={"user_id": victim}) + user = await db.db.litellm_usertable.find_unique(where={"user_id": victim}) assert user is not None and user.teams == [] -async def test_team_bulk_member_delete_row_naming_both_identifiers_is_422(proxy_client, prisma, scratch, world): - victim = scratch.tag("victim") - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id, member_user_ids=[victim]) +async def test_team_bulk_member_delete_row_naming_both_identifiers_is_422(gateway_client, db, resources, actors): + victim = resources.tag("victim") + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id, member_user_ids=[victim]) - resp = await proxy_client.post( - f"/management/v1/teams/{scratch.prefix}/members/bulk_delete", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + resp = await gateway_client.post( + f"/management/v1/teams/{resources.prefix}/members/bulk_delete", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"members": [{"user_id": victim, "user_email": f"{victim}@example.com"}]}, ) assert resp.status_code == 422, resp.text @@ -162,48 +162,48 @@ async def test_team_bulk_member_delete_row_naming_both_identifiers_is_422(proxy_ == "members.0: Value error, Each member must be identified by exactly one of user_id or user_email" ) - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + row = await db.db.litellm_teamtable.find_unique(where={"team_id": resources.prefix}) assert row is not None and victim in _member_ids(row) -async def test_team_bulk_member_delete_unknown_query_param_is_400(proxy_client, prisma, scratch, world): - victim = scratch.tag("victim") - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id, member_user_ids=[victim]) +async def test_team_bulk_member_delete_unknown_query_param_is_400(gateway_client, db, resources, actors): + victim = resources.tag("victim") + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id, member_user_ids=[victim]) - resp = await proxy_client.post( - f"/management/v1/teams/{scratch.prefix}/members/bulk_delete?dry_run=1", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + resp = await gateway_client.post( + f"/management/v1/teams/{resources.prefix}/members/bulk_delete?dry_run=1", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"members": [{"user_id": victim}]}, ) assert resp.status_code == 400, resp.text assert resp.headers["content-type"] == "application/problem+json" assert "dry_run" in resp.json()["detail"] - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + row = await db.db.litellm_teamtable.find_unique(where={"team_id": resources.prefix}) assert row is not None and victim in _member_ids(row) -async def test_team_bulk_member_delete_unknown_body_field_is_422(proxy_client, prisma, scratch, world): - victim = scratch.tag("victim") - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id, member_user_ids=[victim]) +async def test_team_bulk_member_delete_unknown_body_field_is_422(gateway_client, db, resources, actors): + victim = resources.tag("victim") + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id, member_user_ids=[victim]) - resp = await proxy_client.post( - f"/management/v1/teams/{scratch.prefix}/members/bulk_delete", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - json={"team_id": scratch.prefix, "members": [{"user_id": victim}]}, + resp = await gateway_client.post( + f"/management/v1/teams/{resources.prefix}/members/bulk_delete", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"team_id": resources.prefix, "members": [{"user_id": victim}]}, ) assert resp.status_code == 422, resp.text assert "team_id" in resp.json()["detail"] - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + row = await db.db.litellm_teamtable.find_unique(where={"team_id": resources.prefix}) assert row is not None and victim in _member_ids(row) -async def test_team_bulk_member_delete_unknown_team_is_404_problem(proxy_client, scratch, world): - resp = await proxy_client.post( - f"/management/v1/teams/{scratch.tag('missing')}/members/bulk_delete", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - json={"members": [{"user_id": scratch.tag("victim")}]}, +async def test_team_bulk_member_delete_unknown_team_is_404_problem(gateway_client, resources, actors): + resp = await gateway_client.post( + f"/management/v1/teams/{resources.tag('missing')}/members/bulk_delete", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"members": [{"user_id": resources.tag("victim")}]}, ) assert resp.status_code == 404, resp.text assert resp.headers["content-type"] == "application/problem+json" diff --git a/tests/proxy_behavior/management/test_team_daily_activity.py b/tests/integration/management/test_team_daily_activity.py similarity index 61% rename from tests/proxy_behavior/management/test_team_daily_activity.py rename to tests/integration/management/test_team_daily_activity.py index d84cc4c94af..1b71db993ef 100644 --- a/tests/proxy_behavior/management/test_team_daily_activity.py +++ b/tests/integration/management/test_team_daily_activity.py @@ -1,6 +1,9 @@ +from typing import Final + import pytest -from .actors import Actor +from tests.integration._support.actors import Actor +from tests.integration._support.database import write_rows pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -41,6 +44,21 @@ _CASES = [ _DATES = "start_date=2024-01-01&end_date=2024-12-31" +_SEEDED_SPEND: Final = {"alpha": 1.25, "beta": 2.5} + + +@pytest.fixture(scope="module") +def seeded_team_spend(actors) -> None: + team_ids: Final = {"alpha": actors.team_alpha_id, "beta": actors.team_beta_id} + for team, spend in _SEEDED_SPEND.items(): + write_rows( + 'INSERT INTO "LiteLLM_DailyTeamSpend" (id, team_id, date, api_key, model, spend, updated_at) ' + "VALUES (%s, %s, '2024-06-01', 'integration-daily-activity', 'gpt-4o', %s, now()) " + "ON CONFLICT (id) DO NOTHING", + (f"integration-daily-activity-{team}", team_ids[team], str(spend)), + ) + + @pytest.mark.parametrize( "endpoint", ("/team/daily/activity", "/team/daily/activity/aggregated"), @@ -52,18 +70,20 @@ _DATES = "start_date=2024-01-01&end_date=2024-12-31" ids=[c[0] for c in _CASES], ) async def test_team_daily_activity_matrix( - actor: Actor, team: str, expected_status: int, endpoint: str, proxy_client, world + actor: Actor, team: str, expected_status: int, endpoint: str, gateway_client, actors, seeded_team_spend ): query = _DATES if team == "alpha": - query += f"&team_ids={world.team_alpha_id}" + query += f"&team_ids={actors.team_alpha_id}" elif team == "beta": - query += f"&team_ids={world.team_beta_id}" + query += f"&team_ids={actors.team_beta_id}" - resp = await proxy_client.get( + resp = await gateway_client.get( f"{endpoint}?{query}", - headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[actor].cleartext}"}, ) assert ( resp.status_code == expected_status ), f"{actor.value} -> {team}: {resp.status_code} {resp.text}" + if actor == Actor.PROXY_ADMIN and team in _SEEDED_SPEND: + assert resp.json()["metadata"]["total_spend"] == _SEEDED_SPEND[team], resp.text diff --git a/tests/proxy_behavior/management/test_team_delete.py b/tests/integration/management/test_team_delete.py similarity index 66% rename from tests/proxy_behavior/management/test_team_delete.py rename to tests/integration/management/test_team_delete.py index 2fa1ba09883..a4b3a4121b5 100644 --- a/tests/proxy_behavior/management/test_team_delete.py +++ b/tests/integration/management/test_team_delete.py @@ -1,7 +1,7 @@ import pytest -from .actors import Actor -from .conftest import create_scratch_team +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_team pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -33,26 +33,26 @@ async def test_team_delete_authz_matrix( actor: Actor, shape: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - org_id = world.org_a_id if shape == "alpha" else world.org_b_id - await create_scratch_team(prisma, scratch.prefix, organization_id=org_id) - caller = world.keys[actor] + org_id = actors.org_a_id if shape == "alpha" else actors.org_b_id + await create_scratch_team(db, resources.prefix, organization_id=org_id) + caller = actors.keys[actor] - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/delete", headers={"Authorization": f"Bearer {caller.cleartext}"}, - json={"team_ids": [scratch.prefix], "organization_id": org_id}, + json={"team_ids": [resources.prefix], "organization_id": org_id}, ) assert ( resp.status_code == expected_status ), f"{actor.value} {shape}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) if expected_status == 200: assert row is None, "deleted but team row survives" @@ -61,18 +61,18 @@ async def test_team_delete_authz_matrix( async def test_team_delete_batch_with_missing_id_deletes_nothing( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """A batch is validated whole before any deletion: one missing team_id fails the request 404 and the accessible team in the batch survives.""" - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id) - resp = await proxy_client.post( + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id) + resp = await gateway_client.post( "/team/delete", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - json={"team_ids": [scratch.prefix, "behavior-pin-no-such-team"]}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"team_ids": [resources.prefix, "behavior-pin-no-such-team"]}, ) assert resp.status_code == 404, resp.text - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) assert row is not None, "batch aborted but the accessible team was deleted" diff --git a/tests/proxy_behavior/management/test_team_filter_ui.py b/tests/integration/management/test_team_filter_ui.py similarity index 67% rename from tests/proxy_behavior/management/test_team_filter_ui.py rename to tests/integration/management/test_team_filter_ui.py index 69cbabf72a1..8b2e35c0ebc 100644 --- a/tests/proxy_behavior/management/test_team_filter_ui.py +++ b/tests/integration/management/test_team_filter_ui.py @@ -1,6 +1,6 @@ import pytest -from .actors import Actor +from tests.integration._support.actors import Actor pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -12,10 +12,10 @@ pytestmark = pytest.mark.asyncio(loop_scope="session") # route) and 401s every non-proxy-admin before the handler runs. PROXY_ADMIN # reaches the unscoped find_many and sees teams across every org. @pytest.mark.parametrize("actor", list(Actor), ids=[a.value for a in Actor]) -async def test_team_filter_ui_is_proxy_admin_only(actor: Actor, proxy_client, world): - resp = await proxy_client.get( +async def test_team_filter_ui_is_proxy_admin_only(actor: Actor, gateway_client, actors): + resp = await gateway_client.get( "/team/filter/ui", - headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[actor].cleartext}"}, ) expected = 200 if actor == Actor.PROXY_ADMIN else 401 assert ( @@ -23,17 +23,17 @@ async def test_team_filter_ui_is_proxy_admin_only(actor: Actor, proxy_client, wo ), f"{actor.value}: {resp.status_code} {resp.text}" -async def test_team_filter_ui_proxy_admin_sees_cross_org_teams(proxy_client, world): +async def test_team_filter_ui_proxy_admin_sees_cross_org_teams(gateway_client, actors): """The handler runs an unscoped query — PROXY_ADMIN sees teams from every - org, including the three seeded world teams.""" - resp = await proxy_client.get( + org, including the three seeded actors teams.""" + resp = await gateway_client.get( "/team/filter/ui", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, ) assert resp.status_code == 200, resp.text team_ids = {t.get("team_id") for t in resp.json() if isinstance(t, dict)} assert { - world.team_alpha_id, - world.team_beta_id, - world.team_gamma_id, + actors.team_alpha_id, + actors.team_beta_id, + actors.team_gamma_id, } <= team_ids diff --git a/tests/proxy_behavior/management/test_team_info.py b/tests/integration/management/test_team_info.py similarity index 85% rename from tests/proxy_behavior/management/test_team_info.py rename to tests/integration/management/test_team_info.py index eecb22cf731..bad6ccb3ceb 100644 --- a/tests/proxy_behavior/management/test_team_info.py +++ b/tests/integration/management/test_team_info.py @@ -1,6 +1,6 @@ import pytest -from .actors import Actor +from tests.integration._support.actors import Actor pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -47,16 +47,16 @@ _SCENARIOS = [ ids=[s[0] for s in _SCENARIOS], ) async def test_team_info_authz_matrix( - actor: Actor, target: str, expected_status: int, proxy_client, world + actor: Actor, target: str, expected_status: int, gateway_client, actors ): - caller = world.keys[actor] + caller = actors.keys[actor] target_team_id = { - "alpha": world.team_alpha_id, - "gamma": world.team_gamma_id, - "beta": world.team_beta_id, + "alpha": actors.team_alpha_id, + "gamma": actors.team_gamma_id, + "beta": actors.team_beta_id, }[target] - resp = await proxy_client.get( + resp = await gateway_client.get( f"/team/info?team_id={target_team_id}", headers={"Authorization": f"Bearer {caller.cleartext}"}, ) @@ -74,10 +74,10 @@ async def test_team_info_authz_matrix( # alpha/org_b_admin already covers the branch in the matrix; this guard # turns a silent rename of the exception detail into a CI red, which is the # behavior tripwire that the matrix's status-only assertion cannot catch. -async def test_team_info_org_admin_cross_org_rejection_detail(proxy_client, world): - resp = await proxy_client.get( - f"/team/info?team_id={world.team_alpha_id}", - headers={"Authorization": f"Bearer {world.keys[Actor.ORG_B_ADMIN].cleartext}"}, +async def test_team_info_org_admin_cross_org_rejection_detail(gateway_client, actors): + resp = await gateway_client.get( + f"/team/info?team_id={actors.team_alpha_id}", + headers={"Authorization": f"Bearer {actors.keys[Actor.ORG_B_ADMIN].cleartext}"}, ) assert resp.status_code == 403, resp.text # validate_membership() at GET /team/info raises with this exact phrase. diff --git a/tests/proxy_behavior/management/test_team_key_bulk_update.py b/tests/integration/management/test_team_key_bulk_update.py similarity index 68% rename from tests/proxy_behavior/management/test_team_key_bulk_update.py rename to tests/integration/management/test_team_key_bulk_update.py index 5acf0c8185c..8ad3d3db6ee 100644 --- a/tests/proxy_behavior/management/test_team_key_bulk_update.py +++ b/tests/integration/management/test_team_key_bulk_update.py @@ -5,8 +5,8 @@ import pytest from litellm.proxy._types import KeyManagementRoutes from litellm.proxy.utils import hash_token -from .actors import Actor -from .conftest import create_scratch_key, create_scratch_team +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_key, create_scratch_team pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -31,20 +31,20 @@ _MATRIX = [ ] -async def _seed_team_key(prisma, proxy_client, prefix: str, world, shape: str) -> str: - """Raw-seed the scratch team for `shape`, return a team key's cleartext.""" - internal = world.keys[Actor.INTERNAL_USER].user_id - owner = world.keys[Actor.OWNER].user_id +async def _seed_team_key(db, gateway_client, prefix: str, actors, shape: str) -> str: + """Raw-seed the resources team for `shape`, return a team key's cleartext.""" + internal = actors.keys[Actor.INTERNAL_USER].user_id + owner = actors.keys[Actor.OWNER].user_id if shape == "admin": await create_scratch_team( - prisma, prefix, organization_id=world.org_a_id, admin_user_ids=[internal] + db, prefix, organization_id=actors.org_a_id, admin_user_ids=[internal] ) key_owner = internal elif shape == "member_allowed": await create_scratch_team( - prisma, + db, prefix, - organization_id=world.org_a_id, + organization_id=actors.org_a_id, admin_user_ids=[owner], member_user_ids=[internal], team_member_permissions=[_KEY_UPDATE], @@ -52,9 +52,9 @@ async def _seed_team_key(prisma, proxy_client, prefix: str, world, shape: str) - key_owner = owner elif shape == "member_denied": await create_scratch_team( - prisma, + db, prefix, - organization_id=world.org_a_id, + organization_id=actors.org_a_id, admin_user_ids=[owner], member_user_ids=[internal], team_member_permissions=[], @@ -62,14 +62,14 @@ async def _seed_team_key(prisma, proxy_client, prefix: str, world, shape: str) - key_owner = owner elif shape == "nonmember": await create_scratch_team( - prisma, prefix, organization_id=world.org_a_id, admin_user_ids=[owner] + db, prefix, organization_id=actors.org_a_id, admin_user_ids=[owner] ) key_owner = owner else: pytest.fail(f"unknown shape={shape}") # pragma: no cover return await create_scratch_key( - proxy_client, - world.keys[Actor.PROXY_ADMIN].cleartext, + gateway_client, + actors.keys[Actor.PROXY_ADMIN].cleartext, prefix, user_id=key_owner, team_id=prefix, @@ -85,20 +85,20 @@ async def test_team_key_bulk_update_authz_matrix( actor: Actor, shape: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - key = await _seed_team_key(prisma, proxy_client, scratch.prefix, world, shape) + key = await _seed_team_key(db, gateway_client, resources.prefix, actors, shape) hashed = hash_token(key) - caller = world.keys[actor] + caller = actors.keys[actor] - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/key/bulk_update", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={ - "team_id": scratch.prefix, + "team_id": resources.prefix, "key_ids": [key], "update_fields": {"max_budget": _MARKER_BUDGET}, }, @@ -107,7 +107,7 @@ async def test_team_key_bulk_update_authz_matrix( resp.status_code == expected_status ), f"{actor.value} {shape}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_verificationtoken.find_unique(where={"token": hashed}) + row = await db.db.litellm_verificationtoken.find_unique(where={"token": hashed}) assert row is not None if expected_status == 200: assert len(resp.json()["successful_updates"]) == 1 @@ -117,12 +117,12 @@ async def test_team_key_bulk_update_authz_matrix( async def test_team_key_bulk_update_requires_team_id( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """An empty team_id is rejected 400.""" - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/key/bulk_update", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={ "team_id": "", "key_ids": ["sk-" + uuid.uuid4().hex], @@ -133,28 +133,28 @@ async def test_team_key_bulk_update_requires_team_id( async def test_team_key_bulk_update_all_keys_in_team( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """all_keys_in_team=True broadcasts the update to every key in the team.""" - admin = world.keys[Actor.PROXY_ADMIN].cleartext - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id) + admin = actors.keys[Actor.PROXY_ADMIN].cleartext + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id) keys = [ await create_scratch_key( - proxy_client, + gateway_client, admin, - scratch.prefix, - user_id=world.keys[Actor.OWNER].user_id, - team_id=scratch.prefix, - key_alias=f"{scratch.prefix}-k{i}", + resources.prefix, + user_id=actors.keys[Actor.OWNER].user_id, + team_id=resources.prefix, + key_alias=f"{resources.prefix}-k{i}", ) for i in range(2) ] - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/key/bulk_update", headers={"Authorization": f"Bearer {admin}"}, json={ - "team_id": scratch.prefix, + "team_id": resources.prefix, "all_keys_in_team": True, "update_fields": {"max_budget": _MARKER_BUDGET}, }, @@ -162,22 +162,22 @@ async def test_team_key_bulk_update_all_keys_in_team( assert resp.status_code == 200, resp.text assert len(resp.json()["successful_updates"]) == 2 for key in keys: - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": hash_token(key)} ) assert row is not None and row.max_budget == _MARKER_BUDGET async def test_team_key_bulk_update_no_keys_found_is_404( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """all_keys_in_team=True on a team with no keys is a top-level 404.""" - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id) - resp = await proxy_client.post( + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id) + resp = await gateway_client.post( "/team/key/bulk_update", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={ - "team_id": scratch.prefix, + "team_id": resources.prefix, "all_keys_in_team": True, "update_fields": {"max_budget": _MARKER_BUDGET}, }, @@ -186,21 +186,21 @@ async def test_team_key_bulk_update_no_keys_found_is_404( async def test_team_key_bulk_update_missing_key_is_isolated( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """A key_id absent from the team lands in failed_updates; the batch still returns 200 and the real key is updated.""" - admin = world.keys[Actor.PROXY_ADMIN].cleartext + admin = actors.keys[Actor.PROXY_ADMIN].cleartext real = await _seed_team_key( - prisma, proxy_client, scratch.prefix, world, "nonmember" + db, gateway_client, resources.prefix, actors, "nonmember" ) missing = "sk-" + uuid.uuid4().hex - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/key/bulk_update", headers={"Authorization": f"Bearer {admin}"}, json={ - "team_id": scratch.prefix, + "team_id": resources.prefix, "key_ids": [real, missing], "update_fields": {"max_budget": _MARKER_BUDGET}, }, @@ -211,7 +211,7 @@ async def test_team_key_bulk_update_missing_key_is_isolated( assert len(body["successful_updates"]) == 1 assert len(body["failed_updates"]) == 1 - row = await prisma.db.litellm_verificationtoken.find_unique( + row = await db.db.litellm_verificationtoken.find_unique( where={"token": hash_token(real)} ) assert row is not None and row.max_budget == _MARKER_BUDGET diff --git a/tests/proxy_behavior/management/test_team_list.py b/tests/integration/management/test_team_list.py similarity index 84% rename from tests/proxy_behavior/management/test_team_list.py rename to tests/integration/management/test_team_list.py index 2bd106dd2d0..df8660b6502 100644 --- a/tests/proxy_behavior/management/test_team_list.py +++ b/tests/integration/management/test_team_list.py @@ -2,18 +2,18 @@ from typing import FrozenSet, Optional import pytest -from .actors import Actor +from tests.integration._support.actors import Actor pytestmark = pytest.mark.asyncio(loop_scope="session") # The behavior DB may hold teams beyond the three seeded ones, so every # assertion intersects the returned team_ids with the known seeded set. -def _seeded_visible(resp_json, world) -> set: +def _seeded_visible(resp_json, actors) -> set: known = { - world.team_alpha_id: "alpha", - world.team_beta_id: "beta", - world.team_gamma_id: "gamma", + actors.team_alpha_id: "alpha", + actors.team_beta_id: "beta", + actors.team_gamma_id: "gamma", } return { known[entry["team_id"]] @@ -47,11 +47,11 @@ async def test_team_list_bare_authz( actor: Actor, expected_status: int, expected_visible: Optional[set], - proxy_client, - world, + gateway_client, + actors, ): - caller = world.keys[actor] - resp = await proxy_client.get( + caller = actors.keys[actor] + resp = await gateway_client.get( "/team/list", headers={"Authorization": f"Bearer {caller.cleartext}"}, ) @@ -60,7 +60,7 @@ async def test_team_list_bare_authz( ), f"{actor.value}: {resp.status_code} {resp.text}" if expected_status == 200: - visible = _seeded_visible(resp.json(), world) + visible = _seeded_visible(resp.json(), actors) assert visible == expected_visible, ( f"{actor.value}: expected {sorted(expected_visible)}, " f"got {sorted(visible)}" @@ -90,16 +90,16 @@ _OWN = { ids=[a.value for a in _OWN], ) async def test_team_list_own_query( - actor: Actor, expected_visible: FrozenSet[str], proxy_client, world + actor: Actor, expected_visible: FrozenSet[str], gateway_client, actors ): - caller = world.keys[actor] - resp = await proxy_client.get( + caller = actors.keys[actor] + resp = await gateway_client.get( f"/team/list?user_id={caller.user_id}", headers={"Authorization": f"Bearer {caller.cleartext}"}, ) assert resp.status_code == 200, f"{actor.value}: {resp.status_code} {resp.text}" - visible = _seeded_visible(resp.json(), world) + visible = _seeded_visible(resp.json(), actors) assert visible == set(expected_visible), ( f"{actor.value}: expected {sorted(expected_visible)}, " f"got {sorted(visible)}" ) diff --git a/tests/proxy_behavior/management/test_team_list_v2.py b/tests/integration/management/test_team_list_v2.py similarity index 71% rename from tests/proxy_behavior/management/test_team_list_v2.py rename to tests/integration/management/test_team_list_v2.py index 81178ad73c0..d608d054daa 100644 --- a/tests/proxy_behavior/management/test_team_list_v2.py +++ b/tests/integration/management/test_team_list_v2.py @@ -2,26 +2,26 @@ from typing import FrozenSet, Optional import pytest -from .actors import Actor +from tests.integration._support.actors import Actor pytestmark = pytest.mark.asyncio(loop_scope="session") -def _seeded(team_ids: set, world) -> set: +def _seeded(team_ids: set, actors) -> set: known = { - world.team_alpha_id: "alpha", - world.team_beta_id: "beta", - world.team_gamma_id: "gamma", + actors.team_alpha_id: "alpha", + actors.team_beta_id: "beta", + actors.team_gamma_id: "gamma", } return {known[t] for t in team_ids if t in known} -async def _v2_team_ids(proxy_client, caller_cleartext: str, extra: str = "") -> set: +async def _v2_team_ids(gateway_client, caller_cleartext: str, extra: str = "") -> set: """Walk every /v2/team/list page and collect the returned team_ids.""" ids: set = set() page = 1 while True: - resp = await proxy_client.get( + resp = await gateway_client.get( f"/v2/team/list?page={page}&page_size=100{extra}", headers={"Authorization": f"Bearer {caller_cleartext}"}, ) @@ -64,19 +64,19 @@ async def test_team_list_v2_bare( actor: Actor, expected_status: int, expected_visible: Optional[FrozenSet[str]], - proxy_client, - world, + gateway_client, + actors, ): - caller = world.keys[actor] + caller = actors.keys[actor] if expected_status != 200: - resp = await proxy_client.get( + resp = await gateway_client.get( "/v2/team/list", headers={"Authorization": f"Bearer {caller.cleartext}"}, ) assert resp.status_code == expected_status, resp.text return - visible = _seeded(await _v2_team_ids(proxy_client, caller.cleartext), world) + visible = _seeded(await _v2_team_ids(gateway_client, caller.cleartext), actors) assert visible == set( expected_visible ), f"{actor.value}: expected {sorted(expected_visible)}, got {sorted(visible)}" @@ -98,44 +98,44 @@ _OWN = { "actor,expected_visible", list(_OWN.items()), ids=[a.value for a in _OWN] ) async def test_team_list_v2_own_user_id_query( - actor: Actor, expected_visible: FrozenSet[str], proxy_client, world + actor: Actor, expected_visible: FrozenSet[str], gateway_client, actors ): - caller = world.keys[actor] + caller = actors.keys[actor] visible = _seeded( await _v2_team_ids( - proxy_client, caller.cleartext, f"&user_id={caller.user_id}" + gateway_client, caller.cleartext, f"&user_id={caller.user_id}" ), - world, + actors, ) assert visible == set( expected_visible ), f"{actor.value}: expected {sorted(expected_visible)}, got {sorted(visible)}" -async def test_team_list_v2_user_id_filter_other_user_is_401(proxy_client, world): +async def test_team_list_v2_user_id_filter_other_user_is_401(gateway_client, actors): """A regular user filtering by another user's user_id is rejected 401.""" - resp = await proxy_client.get( - f"/v2/team/list?user_id={world.keys[Actor.OWNER].user_id}", + resp = await gateway_client.get( + f"/v2/team/list?user_id={actors.keys[Actor.OWNER].user_id}", headers={ - "Authorization": f"Bearer {world.keys[Actor.INTERNAL_USER].cleartext}" + "Authorization": f"Bearer {actors.keys[Actor.INTERNAL_USER].cleartext}" }, ) assert resp.status_code == 401, resp.text -async def test_team_list_v2_org_filter_foreign_org_is_403(proxy_client, world): +async def test_team_list_v2_org_filter_foreign_org_is_403(gateway_client, actors): """An org admin filtering by an organization it does not administer is 403.""" - resp = await proxy_client.get( - f"/v2/team/list?organization_id={world.org_b_id}", - headers={"Authorization": f"Bearer {world.keys[Actor.ORG_ADMIN].cleartext}"}, + resp = await gateway_client.get( + f"/v2/team/list?organization_id={actors.org_b_id}", + headers={"Authorization": f"Bearer {actors.keys[Actor.ORG_ADMIN].cleartext}"}, ) assert resp.status_code == 403, resp.text -async def test_team_list_v2_invalid_status_is_400(proxy_client, world): +async def test_team_list_v2_invalid_status_is_400(gateway_client, actors): """status accepts only 'deleted' — any other value is 400.""" - resp = await proxy_client.get( + resp = await gateway_client.get( "/v2/team/list?status=bogus", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, ) assert resp.status_code == 400, resp.text diff --git a/tests/proxy_behavior/management/test_team_member_add.py b/tests/integration/management/test_team_member_add.py similarity index 69% rename from tests/proxy_behavior/management/test_team_member_add.py rename to tests/integration/management/test_team_member_add.py index 3309aa11cb0..3f7c686fca3 100644 --- a/tests/proxy_behavior/management/test_team_member_add.py +++ b/tests/integration/management/test_team_member_add.py @@ -1,8 +1,7 @@ -import litellm import pytest -from .actors import Actor -from .conftest import create_scratch_team, create_scratch_user +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_team, create_scratch_user, owned_settings_client pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -40,16 +39,16 @@ _MATRIX = [ ] -async def _seed_target(prisma, world, shape: str, team_id: str) -> None: +async def _seed_target(db, actors, shape: str, team_id: str) -> None: if shape == "alpha": await create_scratch_team( - prisma, + db, team_id, - organization_id=world.org_a_id, - admin_user_ids=[world.keys[Actor.TEAM_ADMIN].user_id], + organization_id=actors.org_a_id, + admin_user_ids=[actors.keys[Actor.TEAM_ADMIN].user_id], ) elif shape == "beta": - await create_scratch_team(prisma, team_id, organization_id=world.org_b_id) + await create_scratch_team(db, team_id, organization_id=actors.org_b_id) else: # pragma: no cover - guard pytest.fail(f"unknown shape={shape}") @@ -67,22 +66,22 @@ async def test_team_member_add_authz_matrix( actor: Actor, shape: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - await _seed_target(prisma, world, shape, scratch.prefix) - caller = world.keys[actor] + await _seed_target(db, actors, shape, resources.prefix) + caller = actors.keys[actor] new_member_id = await create_scratch_user( - prisma, scratch.prefix, suffix="newmember" + db, resources.prefix, suffix="newmember" ) - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/member_add", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={ - "team_id": scratch.prefix, + "team_id": resources.prefix, "member": {"user_id": new_member_id, "role": "user"}, }, ) @@ -90,8 +89,8 @@ async def test_team_member_add_authz_matrix( resp.status_code == expected_status ), f"{actor.value} {shape}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) assert row is not None if expected_status == 200: @@ -121,28 +120,28 @@ _UNPROVISIONED = [ async def test_team_member_add_unprovisioned_user_id_is_proxy_admin_only( actor: Actor, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - await _seed_target(prisma, world, "alpha", scratch.prefix) - caller = world.keys[actor] + await _seed_target(db, actors, "alpha", resources.prefix) + caller = actors.keys[actor] # Deliberately NOT seeded — create_scratch_user is what the matrix above # calls, and its absence here is the whole scenario. - unprovisioned_id = scratch.tag("unprovisioned") + unprovisioned_id = resources.tag("unprovisioned") assert ( - await prisma.db.litellm_usertable.find_unique( + await db.db.litellm_usertable.find_unique( where={"user_id": unprovisioned_id} ) is None ), "setup: user must not exist yet" - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/member_add", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={ - "team_id": scratch.prefix, + "team_id": resources.prefix, "member": {"user_id": unprovisioned_id, "role": "user"}, }, ) @@ -150,11 +149,11 @@ async def test_team_member_add_unprovisioned_user_id_is_proxy_admin_only( resp.status_code == expected_status ), f"{actor.value}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) assert row is not None - created = await prisma.db.litellm_usertable.find_unique( + created = await db.db.litellm_usertable.find_unique( where={"user_id": unprovisioned_id} ) if expected_status == 200: @@ -167,10 +166,10 @@ async def test_team_member_add_unprovisioned_user_id_is_proxy_admin_only( async def test_team_member_add_email_invite_open_to_team_admin( - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): """The escape hatch the provisioning 403 names must actually work. @@ -179,34 +178,31 @@ async def test_team_member_add_email_invite_open_to_team_admin( it stays open to team admins; if it ever closed, the 403 would be sending callers down a dead end. """ - await _seed_target(prisma, world, "alpha", scratch.prefix) - caller = world.keys[Actor.TEAM_ADMIN] - email = f"{scratch.prefix}-invitee@example.com" + await _seed_target(db, actors, "alpha", resources.prefix) + caller = actors.keys[Actor.TEAM_ADMIN] + email = f"{resources.prefix}-invitee@example.com" - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/member_add", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={ - "team_id": scratch.prefix, + "team_id": resources.prefix, "member": {"user_email": email, "role": "user"}, }, ) assert resp.status_code == 200, resp.text - invited = await prisma.db.litellm_usertable.find_first( + invited = await db.db.litellm_usertable.find_first( where={"user_email": email} ) assert invited is not None, "invite did not create the user" - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) assert row is not None assert invited.user_id in _member_ids(row) -# Available-team self-join: a non-admin caller may add ITSELF to a team listed -# in litellm.default_internal_user_params["available_teams"], but the bypass -# must not escalate to role=admin or inject another user. _SELF_JOIN = [ ("self_as_user", "self", "user", 200), ("self_as_admin", "self", "admin", 403), @@ -223,36 +219,33 @@ async def test_team_member_add_available_team_self_join( who: str, role: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, - monkeypatch, + gateway_client, + db, + resources, + actors, + tmp_path, ): - # Org-less team with no admins: the INTERNAL_USER caller is neither team - # nor org admin, so it lands on the available-team branch. - await create_scratch_team(prisma, scratch.prefix) - monkeypatch.setattr( - litellm, "default_internal_user_params", {"available_teams": [scratch.prefix]} - ) + await create_scratch_team(db, resources.prefix) + caller = actors.keys[Actor.INTERNAL_USER] + member_id = caller.user_id if who == "self" else actors.keys[Actor.OWNER].user_id - caller = world.keys[Actor.INTERNAL_USER] - member_id = caller.user_id if who == "self" else world.keys[Actor.OWNER].user_id - - resp = await proxy_client.post( - "/team/member_add", - headers={"Authorization": f"Bearer {caller.cleartext}"}, - json={ - "team_id": scratch.prefix, - "member": {"user_id": member_id, "role": role}, - }, - ) + with owned_settings_client( + gateway_client, tmp_path, {"default_internal_user_params": {"available_teams": [resources.prefix]}} + ) as client: + resp = await client.post( + "/team/member_add", + headers={"Authorization": f"Bearer {caller.cleartext}"}, + json={ + "team_id": resources.prefix, + "member": {"user_id": member_id, "role": role}, + }, + ) assert ( resp.status_code == expected_status ), f"{who}/{role}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) assert row is not None if expected_status == 200: diff --git a/tests/proxy_behavior/management/test_team_member_delete.py b/tests/integration/management/test_team_member_delete.py similarity index 73% rename from tests/proxy_behavior/management/test_team_member_delete.py rename to tests/integration/management/test_team_member_delete.py index 43879d9fd16..47e49e682cd 100644 --- a/tests/proxy_behavior/management/test_team_member_delete.py +++ b/tests/integration/management/test_team_member_delete.py @@ -1,12 +1,12 @@ import pytest -from .actors import Actor -from .conftest import create_scratch_team +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_team pytestmark = pytest.mark.asyncio(loop_scope="session") -# POST /team/member_delete — actor x team-shape matrix. The scratch team is +# POST /team/member_delete — actor x team-shape matrix. The resources team is # raw-seeded with a victim member already in it; PROXY_ADMIN, the team's team # admin, or an org admin of the team's org may remove members; else 403. _MATRIX = [ @@ -31,20 +31,20 @@ _MATRIX = [ ] -async def _seed_target(prisma, world, shape: str, team_id: str, victim_id: str) -> None: +async def _seed_target(db, actors, shape: str, team_id: str, victim_id: str) -> None: if shape == "alpha": await create_scratch_team( - prisma, + db, team_id, - organization_id=world.org_a_id, - admin_user_ids=[world.keys[Actor.TEAM_ADMIN].user_id], + organization_id=actors.org_a_id, + admin_user_ids=[actors.keys[Actor.TEAM_ADMIN].user_id], member_user_ids=[victim_id], ) elif shape == "beta": await create_scratch_team( - prisma, + db, team_id, - organization_id=world.org_b_id, + organization_id=actors.org_b_id, member_user_ids=[victim_id], ) else: # pragma: no cover - guard @@ -64,26 +64,26 @@ async def test_team_member_delete_authz_matrix( actor: Actor, shape: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - victim_id = scratch.tag("victim") - await _seed_target(prisma, world, shape, scratch.prefix, victim_id) - caller = world.keys[actor] + victim_id = resources.tag("victim") + await _seed_target(db, actors, shape, resources.prefix, victim_id) + caller = actors.keys[actor] - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/member_delete", headers={"Authorization": f"Bearer {caller.cleartext}"}, - json={"team_id": scratch.prefix, "user_id": victim_id}, + json={"team_id": resources.prefix, "user_id": victim_id}, ) assert ( resp.status_code == expected_status ), f"{actor.value} {shape}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) assert row is not None if expected_status == 200: diff --git a/tests/proxy_behavior/management/test_team_member_info_validation.py b/tests/integration/management/test_team_member_info_validation.py similarity index 62% rename from tests/proxy_behavior/management/test_team_member_info_validation.py rename to tests/integration/management/test_team_member_info_validation.py index b22e3579400..131e5a1637e 100644 --- a/tests/proxy_behavior/management/test_team_member_info_validation.py +++ b/tests/integration/management/test_team_member_info_validation.py @@ -13,8 +13,9 @@ user_id is invisible from response-body alone). import pytest -from .actors import Actor -from .conftest import create_scratch_team, create_scratch_user +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_team, create_scratch_user +from tests.integration._support.database import write_rows pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -24,10 +25,10 @@ pytestmark = pytest.mark.asyncio(loop_scope="session") # --------------------------------------------------------------------------- -async def test_member_add_both_none_rejected(proxy_client, prisma, scratch, world): - team_id = await create_scratch_team(prisma, team_id=scratch.tag("team")) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( +async def test_member_add_both_none_rejected(gateway_client, db, resources, actors): + team_id = await create_scratch_team(db, team_id=resources.tag("team")) + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/team/member_add", headers={"Authorization": f"Bearer {seeder}"}, json={"team_id": team_id, "member": {"role": "user"}}, @@ -36,8 +37,8 @@ async def test_member_add_both_none_rejected(proxy_client, prisma, scratch, worl # (422). Either shape proves the empty-Member payload is rejected before # any membership row is written — pin both. assert resp.status_code in (400, 422), resp.text - rows = await prisma.db.litellm_teammembership.find_many(where={"team_id": team_id}) - assert rows == [], "empty-Member payload leaked a membership row" + rows = await db.db.litellm_teammembership.find_many(where={"team_id": team_id}) + assert all(row.user_id == "default_user_id" for row in rows), "empty-Member payload leaked a membership row" # --------------------------------------------------------------------------- @@ -46,17 +47,17 @@ async def test_member_add_both_none_rejected(proxy_client, prisma, scratch, worl async def test_member_add_email_id_mismatch_rejected( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): - email = f"{scratch.prefix}-mismatch@example.com" + email = f"{resources.prefix}-mismatch@example.com" real_user_id = await create_scratch_user( - prisma, scratch.prefix, suffix="real", user_email=email + db, resources.prefix, suffix="real", user_email=email ) - other_user_id = await create_scratch_user(prisma, scratch.prefix, suffix="other") + other_user_id = await create_scratch_user(db, resources.prefix, suffix="other") assert real_user_id != other_user_id # sanity - team_id = await create_scratch_team(prisma, team_id=scratch.tag("team")) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + team_id = await create_scratch_team(db, team_id=resources.tag("team")) + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/team/member_add", headers={"Authorization": f"Bearer {seeder}"}, json={ @@ -70,8 +71,8 @@ async def test_member_add_email_id_mismatch_rejected( ) assert resp.status_code == 400, resp.text assert "do not belong to the same user" in resp.text, resp.text - rows = await prisma.db.litellm_teammembership.find_many(where={"team_id": team_id}) - assert rows == [], "mismatch payload leaked a membership row" + rows = await db.db.litellm_teammembership.find_many(where={"team_id": team_id}) + assert not any(row.user_id in {real_user_id, other_user_id} for row in rows), "mismatch payload leaked a membership row" # --------------------------------------------------------------------------- @@ -80,15 +81,15 @@ async def test_member_add_email_id_mismatch_rejected( async def test_member_add_email_only_resolves_user_id( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): - email = f"{scratch.prefix}-resolve@example.com" + email = f"{resources.prefix}-resolve@example.com" user_id = await create_scratch_user( - prisma, scratch.prefix, suffix="lookup", user_email=email + db, resources.prefix, suffix="lookup", user_email=email ) - team_id = await create_scratch_team(prisma, team_id=scratch.tag("team")) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + team_id = await create_scratch_team(db, team_id=resources.tag("team")) + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/team/member_add", headers={"Authorization": f"Bearer {seeder}"}, json={ @@ -102,7 +103,7 @@ async def test_member_add_email_only_resolves_user_id( # team's members_with_roles JSON. Re-read that and assert the resolved # user_id landed — the regression shape is "email resolved to the WRONG # user_id and was silently written to members_with_roles". - team_row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id}) + team_row = await db.db.litellm_teamtable.find_unique(where={"team_id": team_id}) assert team_row is not None member_user_ids = [m.get("user_id") for m in team_row.members_with_roles] assert ( @@ -116,15 +117,15 @@ async def test_member_add_email_only_resolves_user_id( async def test_member_add_unknown_user_id_upserted( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): - team_id = await create_scratch_team(prisma, team_id=scratch.tag("team")) - new_user_id = f"{scratch.prefix}-fresh" + team_id = await create_scratch_team(db, team_id=resources.tag("team")) + new_user_id = f"{resources.prefix}-fresh" # Sanity — user does not exist yet. - pre = await prisma.db.litellm_usertable.find_unique(where={"user_id": new_user_id}) + pre = await db.db.litellm_usertable.find_unique(where={"user_id": new_user_id}) assert pre is None - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/team/member_add", headers={"Authorization": f"Bearer {seeder}"}, json={ @@ -133,7 +134,7 @@ async def test_member_add_unknown_user_id_upserted( }, ) assert resp.status_code == 200, resp.text - post = await prisma.db.litellm_usertable.find_unique(where={"user_id": new_user_id}) + post = await db.db.litellm_usertable.find_unique(where={"user_id": new_user_id}) assert post is not None, "user_id was not upserted" # The user row was created with NULL email (the helper returned the # member as-is, no email lookup happened because the user didn't exist). @@ -143,20 +144,24 @@ async def test_member_add_unknown_user_id_upserted( # --------------------------------------------------------------------------- -# Duplicate-email rejection — two scratch users share an email; email-only +# Duplicate-email rejection — two resources users share an email; email-only # add → 400 with "Multiple users found" detail. # --------------------------------------------------------------------------- async def test_member_add_duplicate_email_rejected( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): - email = f"{scratch.prefix}-dup@example.com" - await create_scratch_user(prisma, scratch.prefix, suffix="dup1", user_email=email) - await create_scratch_user(prisma, scratch.prefix, suffix="dup2", user_email=email) - team_id = await create_scratch_team(prisma, team_id=scratch.tag("team")) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + email = f"{resources.prefix}-dup@example.com" + first = await create_scratch_user(db, resources.prefix, suffix="dup1", user_email=email) + write_rows( + 'INSERT INTO "LiteLLM_UserTable" (user_id, user_email, user_role, models, metadata) ' + "VALUES (%s, %s, 'internal_user', '{}', '{}')", + (resources.tag("dup2"), email), + ) + team_id = await create_scratch_team(db, team_id=resources.tag("team")) + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/team/member_add", headers={"Authorization": f"Bearer {seeder}"}, json={ @@ -166,5 +171,7 @@ async def test_member_add_duplicate_email_rejected( ) assert resp.status_code == 400, resp.text assert "Multiple users found" in resp.text, resp.text - rows = await prisma.db.litellm_teammembership.find_many(where={"team_id": team_id}) + rows = await db.db.litellm_teammembership.find_many( + where={"team_id": team_id, "user_id": {"in": [first, resources.tag("dup2")]}} + ) assert rows == [], "duplicate-email payload leaked a membership row" diff --git a/tests/proxy_behavior/management/test_team_member_me.py b/tests/integration/management/test_team_member_me.py similarity index 74% rename from tests/proxy_behavior/management/test_team_member_me.py rename to tests/integration/management/test_team_member_me.py index bfbbe0504ae..c1bee6c35f3 100644 --- a/tests/proxy_behavior/management/test_team_member_me.py +++ b/tests/integration/management/test_team_member_me.py @@ -4,7 +4,7 @@ import pytest from litellm.proxy.utils import hash_token -from .actors import Actor +from tests.integration._support.actors import Actor pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -38,16 +38,16 @@ _CASES = [ ids=[c[0] for c in _CASES], ) async def test_team_member_me_matrix( - actor: Actor, team: str, expected_status: int, proxy_client, world + actor: Actor, team: str, expected_status: int, gateway_client, actors ): team_id = { - "alpha": world.team_alpha_id, - "beta": world.team_beta_id, - "gamma": world.team_gamma_id, + "alpha": actors.team_alpha_id, + "beta": actors.team_beta_id, + "gamma": actors.team_gamma_id, }[team] - caller = world.keys[actor] + caller = actors.keys[actor] - resp = await proxy_client.get( + resp = await gateway_client.get( f"/team/{team_id}/members/me", headers={"Authorization": f"Bearer {caller.cleartext}"}, ) @@ -62,22 +62,22 @@ async def test_team_member_me_matrix( async def test_team_member_me_team_key_without_user_id_is_400( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """A key with no associated user_id (a team / service-account key) cannot resolve 'me' — the caller has no identity to look up — so it is 400.""" cleartext = "sk-" + uuid.uuid4().hex - await prisma.db.litellm_verificationtoken.create( + await db.db.litellm_verificationtoken.create( data={ "token": hash_token(cleartext), - "key_name": f"{scratch.prefix}-teamkey", - "key_alias": f"{scratch.prefix}-teamkey", - "team_id": world.team_alpha_id, + "key_name": f"{resources.prefix}-teamkey", + "key_alias": f"{resources.prefix}-teamkey", + "team_id": actors.team_alpha_id, "models": [], } ) - resp = await proxy_client.get( - f"/team/{world.team_alpha_id}/members/me", + resp = await gateway_client.get( + f"/team/{actors.team_alpha_id}/members/me", headers={"Authorization": f"Bearer {cleartext}"}, ) assert resp.status_code == 400, resp.text diff --git a/tests/integration/management/test_team_member_reset_budget.py b/tests/integration/management/test_team_member_reset_budget.py new file mode 100644 index 00000000000..211d5a617f0 --- /dev/null +++ b/tests/integration/management/test_team_member_reset_budget.py @@ -0,0 +1,200 @@ +import uuid + +import pytest +from prisma import Json + +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_team + +pytestmark = pytest.mark.asyncio(loop_scope="session") + +_SEED_SPEND = 5.0 +_TEAM_DEFAULT_MAX_BUDGET = 100.0 +_CUSTOM_MAX_BUDGET = 50.0 + +_MATRIX = [ + ("alpha/proxy_admin", Actor.PROXY_ADMIN, "alpha", 200), + ("alpha/org_admin", Actor.ORG_ADMIN, "alpha", 200), + ("alpha/team_admin", Actor.TEAM_ADMIN, "alpha", 200), + ("alpha/internal_user", Actor.INTERNAL_USER, "alpha", 403), + ("alpha/owner", Actor.OWNER, "alpha", 403), + ("alpha/unrelated_same_org", Actor.UNRELATED_SAME_ORG, "alpha", 403), + ("alpha/cross_org_user", Actor.CROSS_ORG_USER, "alpha", 403), + ("alpha/service_account", Actor.SERVICE_ACCOUNT, "alpha", 403), + ("alpha/org_b_admin", Actor.ORG_B_ADMIN, "alpha", 403), + ("beta/proxy_admin", Actor.PROXY_ADMIN, "beta", 200), + ("beta/org_admin", Actor.ORG_ADMIN, "beta", 403), + ("beta/team_admin", Actor.TEAM_ADMIN, "beta", 403), + ("beta/org_b_admin", Actor.ORG_B_ADMIN, "beta", 200), +] + + +async def _seed_budget(db, budget_id: str, max_budget: float) -> str: + await db.db.litellm_budgettable.create( + data={ + "budget_id": budget_id, + "max_budget": max_budget, + "created_by": "phase4-resources", + "updated_by": "phase4-resources", + } + ) + return budget_id + + +async def _seed_team_with_default_budget(db, actors, shape: str, team_id: str, resources) -> str: + default_budget_id = await _seed_budget(db, resources.tag("team-default-budget"), _TEAM_DEFAULT_MAX_BUDGET) + if shape == "alpha": + await create_scratch_team( + db, + team_id, + organization_id=actors.org_a_id, + admin_user_ids=[actors.keys[Actor.TEAM_ADMIN].user_id], + ) + elif shape == "beta": + await create_scratch_team(db, team_id, organization_id=actors.org_b_id) + else: + pytest.fail(f"unknown shape={shape}") + await db.db.litellm_teamtable.update( + where={"team_id": team_id}, + data={"metadata": Json({"team_member_budget_id": default_budget_id})}, + ) + return default_budget_id + + +async def _seed_custom_member(db, team_id: str, member_id: str, resources) -> str: + custom_budget_id = await _seed_budget(db, resources.tag("custom-budget"), _CUSTOM_MAX_BUDGET) + await db.db.litellm_teammembership.create( + data={ + "user_id": member_id, + "team_id": team_id, + "spend": _SEED_SPEND, + "litellm_budget_table": {"connect": {"budget_id": custom_budget_id}}, + } + ) + return custom_budget_id + + +async def _membership(db, team_id: str, member_id: str): + row = await db.db.litellm_teammembership.find_unique( + where={"user_id_team_id": {"user_id": member_id, "team_id": team_id}} + ) + assert row is not None + return row + + +@pytest.mark.parametrize( + "actor,shape,expected_status", + [(a, sh, s) for (_id, a, sh, s) in _MATRIX], + ids=[s[0] for s in _MATRIX], +) +async def test_team_member_reset_budget_authz_matrix( + actor: Actor, + shape: str, + expected_status: int, + gateway_client, + db, + resources, + actors, +): + member_id = resources.tag("member") + default_budget_id = await _seed_team_with_default_budget(db, actors, shape, resources.prefix, resources) + custom_budget_id = await _seed_custom_member(db, resources.prefix, member_id, resources) + caller = actors.keys[actor] + + resp = await gateway_client.post( + f"/team/{resources.prefix}/member/{member_id}/reset_budget", + headers={"Authorization": f"Bearer {caller.cleartext}"}, + ) + assert resp.status_code == expected_status, f"{actor.value} {shape}: {resp.status_code} {resp.text}" + + row = await _membership(db, resources.prefix, member_id) + assert row.spend == _SEED_SPEND, "reset_budget must never touch spend" + if expected_status == 200: + assert row.budget_id == default_budget_id + body = resp.json() + assert body["budget_id"] == default_budget_id + assert body["previous_budget_id"] == custom_budget_id + assert body["budget_source"] == "team_default" + else: + assert row.budget_id == custom_budget_id, "denied but budget relinked" + + +async def test_team_member_reset_budget_leaves_shared_default_row_untouched(gateway_client, db, resources, actors): + member_id = resources.tag("member") + default_budget_id = await _seed_team_with_default_budget(db, actors, "alpha", resources.prefix, resources) + await _seed_custom_member(db, resources.prefix, member_id, resources) + + resp = await gateway_client.post( + f"/team/{resources.prefix}/member/{member_id}/reset_budget", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + ) + assert resp.status_code == 200, resp.text + + default_row = await db.db.litellm_budgettable.find_unique(where={"budget_id": default_budget_id}) + assert default_row is not None and default_row.max_budget == _TEAM_DEFAULT_MAX_BUDGET + + info = await gateway_client.get( + f"/team/info?team_id={resources.prefix}", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + ) + assert info.status_code == 200, info.text + memberships = {tm["user_id"]: tm for tm in info.json()["team_memberships"]} + assert memberships[member_id]["budget_source"] == "team_default" + assert memberships[member_id]["litellm_budget_table"]["max_budget"] == _TEAM_DEFAULT_MAX_BUDGET + + +async def test_team_member_reset_budget_without_team_default_detaches_member(gateway_client, db, resources, actors): + member_id = resources.tag("member") + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id) + await _seed_custom_member(db, resources.prefix, member_id, resources) + + resp = await gateway_client.post( + f"/team/{resources.prefix}/member/{member_id}/reset_budget", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + ) + assert resp.status_code == 200, resp.text + assert resp.json()["budget_id"] is None + assert resp.json()["budget_source"] == "none" + + row = await _membership(db, resources.prefix, member_id) + assert row.budget_id is None + assert row.spend == _SEED_SPEND + + +async def test_team_member_reset_budget_with_deleted_team_default_detaches_member(gateway_client, db, resources, actors): + member_id = resources.tag("member") + await create_scratch_team( + db, + resources.prefix, + organization_id=actors.org_a_id, + metadata={"team_member_budget_id": resources.tag("deleted-budget")}, + ) + await _seed_custom_member(db, resources.prefix, member_id, resources) + + resp = await gateway_client.post( + f"/team/{resources.prefix}/member/{member_id}/reset_budget", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + ) + assert resp.status_code == 200, resp.text + assert resp.json()["budget_id"] is None + assert resp.json()["budget_source"] == "none" + + row = await _membership(db, resources.prefix, member_id) + assert row.budget_id is None + + +async def test_team_member_reset_budget_missing_team_is_404(gateway_client, actors): + resp = await gateway_client.post( + f"/team/behavior-pin-no-such-team/member/{uuid.uuid4().hex}/reset_budget", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + ) + assert resp.status_code == 404, resp.text + + +async def test_team_member_reset_budget_missing_membership_is_404(gateway_client, db, resources, actors): + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id) + resp = await gateway_client.post( + f"/team/{resources.prefix}/member/{uuid.uuid4().hex}/reset_budget", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + ) + assert resp.status_code == 404, resp.text diff --git a/tests/proxy_behavior/management/test_team_member_reset_spend.py b/tests/integration/management/test_team_member_reset_spend.py similarity index 58% rename from tests/proxy_behavior/management/test_team_member_reset_spend.py rename to tests/integration/management/test_team_member_reset_spend.py index fa7765ff6c3..5d8be6a5323 100644 --- a/tests/proxy_behavior/management/test_team_member_reset_spend.py +++ b/tests/integration/management/test_team_member_reset_spend.py @@ -2,8 +2,9 @@ import uuid import pytest -from .actors import Actor -from .conftest import create_scratch_team +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_team +from tests.integration._support.database import write_rows pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -32,19 +33,19 @@ _MATRIX = [ ] -async def _seed_target(prisma, world, shape: str, team_id: str, member_id: str) -> None: +async def _seed_target(db, actors, shape: str, team_id: str, member_id: str) -> None: if shape == "alpha": await create_scratch_team( - prisma, + db, team_id, - organization_id=world.org_a_id, - admin_user_ids=[world.keys[Actor.TEAM_ADMIN].user_id], + organization_id=actors.org_a_id, + admin_user_ids=[actors.keys[Actor.TEAM_ADMIN].user_id], ) elif shape == "beta": - await create_scratch_team(prisma, team_id, organization_id=world.org_b_id) + await create_scratch_team(db, team_id, organization_id=actors.org_b_id) else: # pragma: no cover - guard pytest.fail(f"unknown shape={shape}") - await prisma.db.litellm_teammembership.create( + await db.db.litellm_teammembership.create( data={"user_id": member_id, "team_id": team_id, "spend": _SEED_SPEND} ) @@ -58,17 +59,17 @@ async def test_team_member_reset_spend_authz_matrix( actor: Actor, shape: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - member_id = scratch.tag("member") - await _seed_target(prisma, world, shape, scratch.prefix, member_id) - caller = world.keys[actor] + member_id = resources.tag("member") + await _seed_target(db, actors, shape, resources.prefix, member_id) + caller = actors.keys[actor] - resp = await proxy_client.post( - f"/team/{scratch.prefix}/member/{member_id}/reset_spend", + resp = await gateway_client.post( + f"/team/{resources.prefix}/member/{member_id}/reset_spend", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={"reset_to": _RESET_TO}, ) @@ -76,8 +77,8 @@ async def test_team_member_reset_spend_authz_matrix( resp.status_code == expected_status ), f"{actor.value} {shape}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_teammembership.find_unique( - where={"user_id_team_id": {"user_id": member_id, "team_id": scratch.prefix}} + row = await db.db.litellm_teammembership.find_unique( + where={"user_id_team_id": {"user_id": member_id, "team_id": resources.prefix}} ) assert row is not None if expected_status == 200: @@ -86,67 +87,68 @@ async def test_team_member_reset_spend_authz_matrix( assert row.spend == _SEED_SPEND, "denied but spend reset" -async def test_team_member_reset_spend_missing_team_is_404(proxy_client, world): - resp = await proxy_client.post( +async def test_team_member_reset_spend_missing_team_is_404(gateway_client, actors): + resp = await gateway_client.post( f"/team/behavior-pin-no-such-team/member/{uuid.uuid4().hex}/reset_spend", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"reset_to": 0.0}, ) assert resp.status_code == 404, resp.text async def test_team_member_reset_spend_missing_membership_is_404( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """A well-formed team but a user_id with no LiteLLM_TeamMembership row is 404.""" - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id) - resp = await proxy_client.post( - f"/team/{scratch.prefix}/member/{uuid.uuid4().hex}/reset_spend", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id) + resp = await gateway_client.post( + f"/team/{resources.prefix}/member/{uuid.uuid4().hex}/reset_spend", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"reset_to": 0.0}, ) assert resp.status_code == 404, resp.text async def test_team_member_reset_spend_above_current_spend_is_400( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): - member_id = scratch.tag("member") - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id) - await prisma.db.litellm_teammembership.create( - data={"user_id": member_id, "team_id": scratch.prefix, "spend": 1.0} + member_id = resources.tag("member") + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id) + await db.db.litellm_teammembership.create( + data={"user_id": member_id, "team_id": resources.prefix, "spend": 1.0} ) - resp = await proxy_client.post( - f"/team/{scratch.prefix}/member/{member_id}/reset_spend", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + resp = await gateway_client.post( + f"/team/{resources.prefix}/member/{member_id}/reset_spend", + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"reset_to": 5.0}, ) assert resp.status_code == 400, resp.text async def test_team_member_reset_spend_team_admin_cannot_reset_own_spend( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """A team admin targeting their own LiteLLM_TeamMembership row is 403: unchecked, an admin could repeatedly zero their own spend right before it crosses their per-member cap, consuming the shared team budget without the configured limit ever binding.""" - team_admin = world.keys[Actor.TEAM_ADMIN] + team_admin = actors.keys[Actor.TEAM_ADMIN] await create_scratch_team( - prisma, - scratch.prefix, - organization_id=world.org_a_id, + db, + resources.prefix, + organization_id=actors.org_a_id, admin_user_ids=[team_admin.user_id], ) - await prisma.db.litellm_teammembership.create( - data={"user_id": team_admin.user_id, "team_id": scratch.prefix, "spend": _SEED_SPEND} + write_rows( + 'UPDATE "LiteLLM_TeamMembership" SET spend = %s WHERE user_id = %s AND team_id = %s', + (str(_SEED_SPEND), team_admin.user_id, resources.prefix), ) - resp = await proxy_client.post( - f"/team/{scratch.prefix}/member/{team_admin.user_id}/reset_spend", + resp = await gateway_client.post( + f"/team/{resources.prefix}/member/{team_admin.user_id}/reset_spend", headers={"Authorization": f"Bearer {team_admin.cleartext}"}, json={"reset_to": 0.0}, ) assert resp.status_code == 403, resp.text - row = await prisma.db.litellm_teammembership.find_unique( - where={"user_id_team_id": {"user_id": team_admin.user_id, "team_id": scratch.prefix}} + row = await db.db.litellm_teammembership.find_unique( + where={"user_id_team_id": {"user_id": team_admin.user_id, "team_id": resources.prefix}} ) assert row is not None and row.spend == _SEED_SPEND, "denied but spend reset" diff --git a/tests/proxy_behavior/management/test_team_member_update.py b/tests/integration/management/test_team_member_update.py similarity index 74% rename from tests/proxy_behavior/management/test_team_member_update.py rename to tests/integration/management/test_team_member_update.py index 53b245bd1e9..4aaeab906b9 100644 --- a/tests/proxy_behavior/management/test_team_member_update.py +++ b/tests/integration/management/test_team_member_update.py @@ -1,12 +1,12 @@ import pytest -from .actors import Actor -from .conftest import create_scratch_team +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_team pytestmark = pytest.mark.asyncio(loop_scope="session") -# POST /team/member_update — actor x team-shape matrix. The scratch team is +# POST /team/member_update — actor x team-shape matrix. The resources team is # raw-seeded with a "user"-role member; each scenario tries to promote it to # "admin". PROXY_ADMIN, the team's team admin, or an org admin of the team's # org may update members; else 403. (The harness forces premium_user, so the @@ -33,20 +33,20 @@ _MATRIX = [ ] -async def _seed_target(prisma, world, shape: str, team_id: str, member_id: str) -> None: +async def _seed_target(db, actors, shape: str, team_id: str, member_id: str) -> None: if shape == "alpha": await create_scratch_team( - prisma, + db, team_id, - organization_id=world.org_a_id, - admin_user_ids=[world.keys[Actor.TEAM_ADMIN].user_id], + organization_id=actors.org_a_id, + admin_user_ids=[actors.keys[Actor.TEAM_ADMIN].user_id], member_user_ids=[member_id], ) elif shape == "beta": await create_scratch_team( - prisma, + db, team_id, - organization_id=world.org_b_id, + organization_id=actors.org_b_id, member_user_ids=[member_id], ) else: # pragma: no cover - guard @@ -69,26 +69,26 @@ async def test_team_member_update_authz_matrix( actor: Actor, shape: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - member_id = scratch.tag("member") - await _seed_target(prisma, world, shape, scratch.prefix, member_id) - caller = world.keys[actor] + member_id = resources.tag("member") + await _seed_target(db, actors, shape, resources.prefix, member_id) + caller = actors.keys[actor] - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/member_update", headers={"Authorization": f"Bearer {caller.cleartext}"}, - json={"team_id": scratch.prefix, "user_id": member_id, "role": "admin"}, + json={"team_id": resources.prefix, "user_id": member_id, "role": "admin"}, ) assert ( resp.status_code == expected_status ), f"{actor.value} {shape}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) assert row is not None if expected_status == 200: diff --git a/tests/proxy_behavior/management/test_team_metadata_schema.py b/tests/integration/management/test_team_metadata_schema.py similarity index 57% rename from tests/proxy_behavior/management/test_team_metadata_schema.py rename to tests/integration/management/test_team_metadata_schema.py index c1d308eba33..a421a8c0a83 100644 --- a/tests/proxy_behavior/management/test_team_metadata_schema.py +++ b/tests/integration/management/test_team_metadata_schema.py @@ -1,25 +1,25 @@ -"""GET /team/metadata_schema — the behavior world declares no +"""GET /team/metadata_schema — the behavior actors declares no ``general_settings.team_metadata_schema``, so the route is an info route that returns an empty field list to every authenticated actor and 401s without a key. """ import pytest -from .actors import Actor +from tests.integration._support.actors import Actor pytestmark = pytest.mark.asyncio(loop_scope="session") @pytest.mark.parametrize("actor", list(Actor), ids=[a.value for a in Actor]) -async def test_team_metadata_schema_default_is_empty(actor: Actor, proxy_client, world): - resp = await proxy_client.get( +async def test_team_metadata_schema_default_is_empty(actor: Actor, gateway_client, actors): + resp = await gateway_client.get( "/team/metadata_schema", - headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[actor].cleartext}"}, ) assert resp.status_code == 200, f"{actor.value}: {resp.status_code} {resp.text}" assert resp.json() == {"fields": []} -async def test_team_metadata_schema_requires_auth(proxy_client, world): - resp = await proxy_client.get("/team/metadata_schema") +async def test_team_metadata_schema_requires_auth(gateway_client, actors): + resp = await gateway_client.get("/team/metadata_schema", headers={"Authorization": "Bearer invalid"}) assert resp.status_code == 401, resp.text diff --git a/tests/proxy_behavior/management/test_team_model.py b/tests/integration/management/test_team_model.py similarity index 75% rename from tests/proxy_behavior/management/test_team_model.py rename to tests/integration/management/test_team_model.py index 3564e8df83a..51a699df2c7 100644 --- a/tests/proxy_behavior/management/test_team_model.py +++ b/tests/integration/management/test_team_model.py @@ -1,7 +1,7 @@ import pytest -from .actors import Actor -from .conftest import create_scratch_team +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_team pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -37,28 +37,28 @@ async def test_team_model_authz_matrix( route: str, actor: Actor, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): initial = [] if route == "add" else [_MARKER_MODEL] await create_scratch_team( - prisma, scratch.prefix, organization_id=world.org_a_id, models=initial + db, resources.prefix, organization_id=actors.org_a_id, models=initial ) - caller = world.keys[actor] + caller = actors.keys[actor] - resp = await proxy_client.post( + resp = await gateway_client.post( _ROUTE_URL[route], headers={"Authorization": f"Bearer {caller.cleartext}"}, - json={"team_id": scratch.prefix, "models": [_MARKER_MODEL]}, + json={"team_id": resources.prefix, "models": [_MARKER_MODEL]}, ) assert ( resp.status_code == expected_status ), f"{route} {actor.value}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) assert row is not None if expected_status == 200: @@ -68,11 +68,11 @@ async def test_team_model_authz_matrix( @pytest.mark.parametrize("route", ["add", "delete"]) -async def test_team_model_missing_team_is_404(route: str, proxy_client, world): +async def test_team_model_missing_team_is_404(route: str, gateway_client, actors): """A team_id absent from the DB is 404 — the existence check precedes authz.""" - resp = await proxy_client.post( + resp = await gateway_client.post( _ROUTE_URL[route], - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"team_id": "behavior-pin-no-such-team", "models": [_MARKER_MODEL]}, ) assert resp.status_code == 404, resp.text diff --git a/tests/proxy_behavior/management/test_team_new.py b/tests/integration/management/test_team_new.py similarity index 70% rename from tests/proxy_behavior/management/test_team_new.py rename to tests/integration/management/test_team_new.py index 9846566d0b9..4b8f540407d 100644 --- a/tests/proxy_behavior/management/test_team_new.py +++ b/tests/integration/management/test_team_new.py @@ -2,7 +2,7 @@ from typing import Any, Dict import pytest -from .actors import Actor +from tests.integration._support.actors import Actor pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -51,30 +51,30 @@ async def test_team_new_authz_matrix( actor: Actor, org_target: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - caller = world.keys[actor] + caller = actors.keys[actor] org_id = { "none": None, - "org_a": world.org_a_id, - "org_b": world.org_b_id, + "org_a": actors.org_a_id, + "org_b": actors.org_b_id, }[org_target] - body: Dict[str, Any] = {"team_id": scratch.prefix, "team_alias": scratch.prefix} + body: Dict[str, Any] = {"team_id": resources.prefix, "team_alias": resources.prefix} if org_id is not None: body["organization_id"] = org_id - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/new", headers={"Authorization": f"Bearer {caller.cleartext}"}, json=body, ) assert resp.status_code == expected_status, f"{actor.value} org={org_target}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + row = await db.db.litellm_teamtable.find_unique(where={"team_id": resources.prefix}) if expected_status == 200: assert row is not None assert row.organization_id == org_id @@ -82,49 +82,49 @@ async def test_team_new_authz_matrix( assert row is None, f"{actor.value}: denied but team row leaked" -async def test_team_new_rejects_negative_budget(proxy_client, prisma, scratch, world): +async def test_team_new_rejects_negative_budget(gateway_client, db, resources, actors): """Input-validation pin: max_budget < 0 is a 400, no row created.""" - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/new", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - json={"team_id": scratch.prefix, "max_budget": -1}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"team_id": resources.prefix, "max_budget": -1}, ) assert resp.status_code == 400, resp.text - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + row = await db.db.litellm_teamtable.find_unique(where={"team_id": resources.prefix}) assert row is None -async def test_team_new_rejects_duplicate_team_id(proxy_client, prisma, scratch, world): +async def test_team_new_rejects_duplicate_team_id(gateway_client, db, resources, actors): """Input-validation pin: a colliding team_id is a 400 on the second call.""" - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - first = await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + first = await gateway_client.post( "/team/new", headers={"Authorization": f"Bearer {seeder}"}, - json={"team_id": scratch.prefix, "team_alias": scratch.prefix}, + json={"team_id": resources.prefix, "team_alias": resources.prefix}, ) assert first.status_code == 200, first.text - second = await proxy_client.post( + second = await gateway_client.post( "/team/new", headers={"Authorization": f"Bearer {seeder}"}, - json={"team_id": scratch.prefix, "team_alias": scratch.prefix}, + json={"team_id": resources.prefix, "team_alias": resources.prefix}, ) assert second.status_code == 400, second.text -async def test_team_new_unknown_organization_is_400(proxy_client, prisma, scratch, world): +async def test_team_new_unknown_organization_is_400(gateway_client, db, resources, actors): """A /team/new with an organization_id that does not exist fails 400: OrganizationNotFoundError is routed into the handler's own 'Organization not found' guard instead of escaping as a 500.""" - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/new", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={ - "team_id": scratch.prefix, - "organization_id": scratch.tag("no-such-org"), + "team_id": resources.prefix, + "organization_id": resources.tag("no-such-org"), }, ) assert resp.status_code == 400, resp.text assert "Organization not found" in resp.text - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + row = await db.db.litellm_teamtable.find_unique(where={"team_id": resources.prefix}) assert row is None diff --git a/tests/proxy_behavior/management/test_team_permissions.py b/tests/integration/management/test_team_permissions.py similarity index 52% rename from tests/proxy_behavior/management/test_team_permissions.py rename to tests/integration/management/test_team_permissions.py index 5d16702fe6c..4b33082c6d2 100644 --- a/tests/proxy_behavior/management/test_team_permissions.py +++ b/tests/integration/management/test_team_permissions.py @@ -1,10 +1,9 @@ -import litellm import pytest from litellm.proxy._types import KeyManagementRoutes -from .actors import Actor -from .conftest import create_scratch_team +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_team, owned_settings_client pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -13,7 +12,7 @@ _PERM = KeyManagementRoutes.KEY_INFO.value # GET /team/permissions_list and POST /team/permissions_update are self-managed # routes, so every actor reaches the handler. Both grant access to PROXY_ADMIN, -# the team admin, or an org admin of the team's org. The scratch team is in +# the team admin, or an org admin of the team's org. The resources team is in # ORG_A with TEAM_ADMIN as its team admin. _MATRIX = [ ("proxy_admin", Actor.PROXY_ADMIN, 200), @@ -28,17 +27,17 @@ _MATRIX = [ ] -async def _seed_team(prisma, scratch_prefix, world) -> None: +async def _seed_team(db, scratch_prefix, actors) -> None: await create_scratch_team( - prisma, + db, scratch_prefix, - organization_id=world.org_a_id, - admin_user_ids=[world.keys[Actor.TEAM_ADMIN].user_id], + organization_id=actors.org_a_id, + admin_user_ids=[actors.keys[Actor.TEAM_ADMIN].user_id], member_user_ids=[ - world.keys[Actor.INTERNAL_USER].user_id, - world.keys[Actor.OWNER].user_id, - world.keys[Actor.UNRELATED_SAME_ORG].user_id, - world.keys[Actor.SERVICE_ACCOUNT].user_id, + actors.keys[Actor.INTERNAL_USER].user_id, + actors.keys[Actor.OWNER].user_id, + actors.keys[Actor.UNRELATED_SAME_ORG].user_id, + actors.keys[Actor.SERVICE_ACCOUNT].user_id, ], ) @@ -49,18 +48,18 @@ async def _seed_team(prisma, scratch_prefix, world) -> None: ids=[s[0] for s in _MATRIX], ) async def test_team_permissions_list_authz_matrix( - actor: Actor, expected_status: int, proxy_client, prisma, scratch, world + actor: Actor, expected_status: int, gateway_client, db, resources, actors ): - await _seed_team(prisma, scratch.prefix, world) - resp = await proxy_client.get( - f"/team/permissions_list?team_id={scratch.prefix}", - headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"}, + await _seed_team(db, resources.prefix, actors) + resp = await gateway_client.get( + f"/team/permissions_list?team_id={resources.prefix}", + headers={"Authorization": f"Bearer {actors.keys[actor].cleartext}"}, ) assert ( resp.status_code == expected_status ), f"{actor.value}: {resp.status_code} {resp.text}" if expected_status == 200: - assert resp.json()["team_id"] == scratch.prefix + assert resp.json()["team_id"] == resources.prefix @pytest.mark.parametrize( @@ -69,20 +68,20 @@ async def test_team_permissions_list_authz_matrix( ids=[s[0] for s in _MATRIX], ) async def test_team_permissions_update_authz_matrix( - actor: Actor, expected_status: int, proxy_client, prisma, scratch, world + actor: Actor, expected_status: int, gateway_client, db, resources, actors ): - await _seed_team(prisma, scratch.prefix, world) - resp = await proxy_client.post( + await _seed_team(db, resources.prefix, actors) + resp = await gateway_client.post( "/team/permissions_update", - headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"}, - json={"team_id": scratch.prefix, "team_member_permissions": [_PERM]}, + headers={"Authorization": f"Bearer {actors.keys[actor].cleartext}"}, + json={"team_id": resources.prefix, "team_member_permissions": [_PERM]}, ) assert ( resp.status_code == expected_status ), f"{actor.value}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) assert row is not None if expected_status == 200: @@ -92,29 +91,24 @@ async def test_team_permissions_update_authz_matrix( async def test_team_permissions_available_team_self_join_divergence( - proxy_client, prisma, scratch, world, monkeypatch + gateway_client, db, resources, actors, tmp_path ): - """permissions_list honours the available-team self-join — a non-admin can - READ an available team's permissions — but permissions_update deliberately - does not: the same caller is 403 on update. default_internal_user_params is - module-level litellm.* state, so monkeypatch save/restores it.""" - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id) - monkeypatch.setattr( - litellm, "default_internal_user_params", {"available_teams": [scratch.prefix]} - ) - caller = world.keys[Actor.CROSS_ORG_USER] # non-admin, unrelated to the team + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id) + caller = actors.keys[Actor.CROSS_ORG_USER] - listed = await proxy_client.get( - f"/team/permissions_list?team_id={scratch.prefix}", - headers={"Authorization": f"Bearer {caller.cleartext}"}, - ) + with owned_settings_client( + gateway_client, tmp_path, {"default_internal_user_params": {"available_teams": [resources.prefix]}} + ) as client: + listed = await client.get( + f"/team/permissions_list?team_id={resources.prefix}", + headers={"Authorization": f"Bearer {caller.cleartext}"}, + ) + updated = await client.post( + "/team/permissions_update", + headers={"Authorization": f"Bearer {caller.cleartext}"}, + json={"team_id": resources.prefix, "team_member_permissions": [_PERM]}, + ) assert listed.status_code == 200, listed.text - - updated = await proxy_client.post( - "/team/permissions_update", - headers={"Authorization": f"Bearer {caller.cleartext}"}, - json={"team_id": scratch.prefix, "team_member_permissions": [_PERM]}, - ) assert updated.status_code == 403, updated.text @@ -138,20 +132,20 @@ _BULK_MATRIX = [ ids=[s[0] for s in _BULK_MATRIX], ) async def test_team_permissions_bulk_update_authz_matrix( - actor: Actor, expected_status: int, proxy_client, prisma, scratch, world + actor: Actor, expected_status: int, gateway_client, db, resources, actors ): - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id) - resp = await proxy_client.post( + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id) + resp = await gateway_client.post( "/team/permissions_bulk_update", - headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"}, - json={"team_ids": [scratch.prefix], "permissions": [_PERM]}, + headers={"Authorization": f"Bearer {actors.keys[actor].cleartext}"}, + json={"team_ids": [resources.prefix], "permissions": [_PERM]}, ) assert ( resp.status_code == expected_status ), f"{actor.value}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) assert row is not None if expected_status == 200: @@ -160,11 +154,11 @@ async def test_team_permissions_bulk_update_authz_matrix( assert _PERM not in (row.team_member_permissions or []), "denied but mutated" -async def test_team_permissions_bulk_update_no_selector_is_400(proxy_client, world): +async def test_team_permissions_bulk_update_no_selector_is_400(gateway_client, actors): """Neither team_ids nor apply_to_all_teams is a 400.""" - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/permissions_bulk_update", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"permissions": [_PERM]}, ) assert resp.status_code == 400, resp.text diff --git a/tests/proxy_behavior/management/test_team_permissions_bulk_update.py b/tests/integration/management/test_team_permissions_bulk_update.py similarity index 57% rename from tests/proxy_behavior/management/test_team_permissions_bulk_update.py rename to tests/integration/management/test_team_permissions_bulk_update.py index 552c3526eea..365150087d8 100644 --- a/tests/proxy_behavior/management/test_team_permissions_bulk_update.py +++ b/tests/integration/management/test_team_permissions_bulk_update.py @@ -8,22 +8,20 @@ Pins the three helpers The route is admin-only — PROXY_ADMIN is the only legal caller. The contracts under test: * specific-team list → ONLY listed teams mutate; every other team - (including world teams) is byte-identical on re-read. - * apply_to_all_teams → every team gains the permission, idempotently + (including actors teams) is byte-identical on re-read. + * apply_to_all_teams → every team has the permission, idempotently merged (re-running with the same permission is a no-op). * unknown team_id → 404 (the missing_ids guard); no mutation anywhere. * malformed payload (neither / both selector flags) → 400; no mutation. -The all-teams scenario mutates world teams as a deliberate side effect of -the path under test. The test snapshots every team's -`team_member_permissions` up front and restores them on exit so the -read-world stays immutable for downstream tests. """ +from typing import Final + import pytest -from .actors import TEAM_ALPHA, TEAM_BETA, TEAM_GAMMA, Actor -from .conftest import create_scratch_team +from tests.integration._support.actors import TEAM_ALPHA, TEAM_BETA, TEAM_GAMMA, Actor +from tests.integration._support.actors import create_scratch_team pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -34,28 +32,28 @@ pytestmark = pytest.mark.asyncio(loop_scope="session") async def test_bulk_update_specific_team_only_mutates_listed( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): target = await create_scratch_team( - prisma, - team_id=scratch.tag("target"), + db, + team_id=resources.tag("target"), team_member_permissions=[], ) bystander = await create_scratch_team( - prisma, - team_id=scratch.tag("bystander"), + db, + team_id=resources.tag("bystander"), team_member_permissions=["pre-existing-perm"], ) - # Snapshot world teams so we can assert byte-equality on re-read. - world_team_ids = [TEAM_ALPHA, TEAM_BETA, TEAM_GAMMA] + # Snapshot actors teams so we can assert byte-equality on re-read. + world_team_ids = [actors.team_alpha_id, actors.team_beta_id, actors.team_gamma_id] before = {} for tid in world_team_ids + [bystander]: - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": tid}) + row = await db.db.litellm_teamtable.find_unique(where={"team_id": tid}) before[tid] = list(row.team_member_permissions or []) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext perm = "/key/info" - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/permissions_bulk_update", headers={"Authorization": f"Bearer {seeder}"}, json={"team_ids": [target], "permissions": [perm]}, @@ -63,7 +61,7 @@ async def test_bulk_update_specific_team_only_mutates_listed( assert resp.status_code == 200, resp.text assert resp.json()["teams_updated"] == 1 - target_row = await prisma.db.litellm_teamtable.find_unique( + target_row = await db.db.litellm_teamtable.find_unique( where={"team_id": target} ) assert perm in ( @@ -71,7 +69,7 @@ async def test_bulk_update_specific_team_only_mutates_listed( ), f"target team did not gain perm; got={target_row.team_member_permissions}" for tid, before_perms in before.items(): - after_row = await prisma.db.litellm_teamtable.find_unique( + after_row = await db.db.litellm_teamtable.find_unique( where={"team_id": tid} ) assert list(after_row.team_member_permissions or []) == before_perms, ( @@ -87,15 +85,15 @@ async def test_bulk_update_specific_team_only_mutates_listed( async def test_bulk_update_specific_team_idempotent( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): target = await create_scratch_team( - prisma, - team_id=scratch.tag("target"), + db, + team_id=resources.tag("target"), team_member_permissions=["/key/info"], ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/team/permissions_bulk_update", headers={"Authorization": f"Bearer {seeder}"}, json={ @@ -113,25 +111,25 @@ async def test_bulk_update_specific_team_idempotent( async def test_bulk_update_unknown_team_id_rejected( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): existing = await create_scratch_team( - prisma, - team_id=scratch.tag("real"), + db, + team_id=resources.tag("real"), team_member_permissions=[], ) before = list( ( - await prisma.db.litellm_teamtable.find_unique(where={"team_id": existing}) + await db.db.litellm_teamtable.find_unique(where={"team_id": existing}) ).team_member_permissions or [] ) - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/team/permissions_bulk_update", headers={"Authorization": f"Bearer {seeder}"}, json={ - "team_ids": [existing, f"{scratch.prefix}-ghost"], + "team_ids": [existing, f"{resources.prefix}-ghost"], "permissions": ["/key/info"], }, ) @@ -141,7 +139,7 @@ async def test_bulk_update_unknown_team_id_rejected( assert "not found" in resp.text.lower() or "ghost" in resp.text, resp.text # Critical: the partial-success regression shape is "real team got # mutated before the ghost-id check ran". Re-read and assert it didn't. - after_real = await prisma.db.litellm_teamtable.find_unique( + after_real = await db.db.litellm_teamtable.find_unique( where={"team_id": existing} ) assert ( @@ -173,10 +171,10 @@ async def test_bulk_update_unknown_team_id_rejected( ids=["neither_selector", "both_selectors"], ) async def test_bulk_update_selector_validation( - body, expected_substring: str, proxy_client, world + body, expected_substring: str, gateway_client, actors ): - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( "/team/permissions_bulk_update", headers={"Authorization": f"Bearer {seeder}"}, json=body, @@ -186,60 +184,40 @@ async def test_bulk_update_selector_validation( # --------------------------------------------------------------------------- -# apply_to_all_teams — mutates every team. Snapshot + restore world teams -# so the read-world contract holds for downstream tests. +# apply_to_all_teams # --------------------------------------------------------------------------- async def test_bulk_update_apply_to_all_mutates_every_team( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): - # Two scratch teams so we can assert "every" includes our own targets. + # Two resources teams so we can assert "every" includes our own targets. a = await create_scratch_team( - prisma, team_id=scratch.tag("a"), team_member_permissions=[] + db, team_id=resources.tag("a"), team_member_permissions=[] ) b = await create_scratch_team( - prisma, team_id=scratch.tag("b"), team_member_permissions=["other"] + db, team_id=resources.tag("b"), team_member_permissions=["other"] ) perm = "/key/health" # distinct from the specific-team scenarios above - # Snapshot every team's permission list so we can restore world teams. - all_teams = await prisma.db.litellm_teamtable.find_many() + all_teams = await db.db.litellm_teamtable.find_many() snapshot = {t.team_id: list(t.team_member_permissions or []) for t in all_teams} + expected_updates: Final = sum( + 1 for permissions in snapshot.values() if perm not in permissions + ) - try: - seeder = world.keys[Actor.PROXY_ADMIN].cleartext - resp = await proxy_client.post( - "/team/permissions_bulk_update", - headers={"Authorization": f"Bearer {seeder}"}, - json={"apply_to_all_teams": True, "permissions": [perm]}, - ) - assert resp.status_code == 200, resp.text - # `teams_updated` counts teams that didn't already have the perm — - # i.e. every team in the DB at call time. - assert resp.json()["teams_updated"] == len(all_teams) + seeder = actors.keys[Actor.PROXY_ADMIN].cleartext + resp = await gateway_client.post( + "/team/permissions_bulk_update", + headers={"Authorization": f"Bearer {seeder}"}, + json={"apply_to_all_teams": True, "permissions": [perm]}, + ) + assert resp.status_code == 200, resp.text + assert resp.json()["teams_updated"] == expected_updates - post = await prisma.db.litellm_teamtable.find_many() - for team in post: - perms = list(team.team_member_permissions or []) - assert perm in perms, f"team {team.team_id} missing the all-perm: {perms}" - # Existing perms preserved (merge, not replace). - for prior in snapshot.get(team.team_id, []): - assert ( - prior in perms - ), f"team {team.team_id} lost prior perm {prior!r}: {perms}" - finally: - # Restore every team — scratch teardown handles {a, b}; we must - # explicitly restore world teams (and any other non-scratch teams - # that snuck in) so downstream tests see the immutable world. - for team_id, prior_perms in snapshot.items(): - current = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": team_id} - ) - if current is None: - continue - if list(current.team_member_permissions or []) != prior_perms: - await prisma.db.litellm_teamtable.update( - where={"team_id": team_id}, - data={"team_member_permissions": prior_perms}, - ) + post = await db.db.litellm_teamtable.find_many() + for team in post: + perms = list(team.team_member_permissions or []) + assert perm in perms, f"team {team.team_id} missing the all-perm: {perms}" + for prior in snapshot.get(team.team_id, []): + assert prior in perms, f"team {team.team_id} lost prior perm {prior!r}: {perms}" diff --git a/tests/proxy_behavior/management/test_team_spend_by_user.py b/tests/integration/management/test_team_spend_by_user.py similarity index 84% rename from tests/proxy_behavior/management/test_team_spend_by_user.py rename to tests/integration/management/test_team_spend_by_user.py index 1d6aab04003..bd349e23681 100644 --- a/tests/proxy_behavior/management/test_team_spend_by_user.py +++ b/tests/integration/management/test_team_spend_by_user.py @@ -1,6 +1,6 @@ import pytest -from .actors import Actor +from tests.integration._support.actors import Actor pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -43,13 +43,13 @@ _DATES = "start_date=2024-01-01&end_date=2024-12-31" [(a, t, s) for (_id, a, t, s) in _CASES], ids=[c[0] for c in _CASES], ) -async def test_team_spend_by_user_matrix(actor: Actor, team: str, expected_status: int, proxy_client, world): - team_id = {"alpha": world.team_alpha_id, "beta": world.team_beta_id}.get(team) +async def test_team_spend_by_user_matrix(actor: Actor, team: str, expected_status: int, gateway_client, actors): + team_id = {"alpha": actors.team_alpha_id, "beta": actors.team_beta_id}.get(team) query = _DATES if team_id is None else f"{_DATES}&team_ids={team_id}" - resp = await proxy_client.get( + resp = await gateway_client.get( f"/team/spend/by_user?{query}", - headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[actor].cleartext}"}, ) assert resp.status_code == expected_status, f"{actor.value} -> {team}: {resp.status_code} {resp.text}" if expected_status == 200: diff --git a/tests/proxy_behavior/management/test_team_update.py b/tests/integration/management/test_team_update.py similarity index 65% rename from tests/proxy_behavior/management/test_team_update.py rename to tests/integration/management/test_team_update.py index 50d6ec6ccaa..561191c9722 100644 --- a/tests/proxy_behavior/management/test_team_update.py +++ b/tests/integration/management/test_team_update.py @@ -2,8 +2,8 @@ import pytest from litellm.proxy._types import LitellmUserRoles -from .actors import Actor -from .conftest import create_scratch_actor, create_scratch_team +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_actor, create_scratch_team pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -38,30 +38,30 @@ _MATRIX = [ ] -async def _seed_target(prisma, world, shape: str, team_id: str) -> str: - """Raw-seed the scratch target team; returns its organization_id.""" +async def _seed_target(db, actors, shape: str, team_id: str) -> str: + """Raw-seed the resources target team; returns its organization_id.""" if shape == "alpha": await create_scratch_team( - prisma, + db, team_id, - organization_id=world.org_a_id, - admin_user_ids=[world.keys[Actor.TEAM_ADMIN].user_id], + organization_id=actors.org_a_id, + admin_user_ids=[actors.keys[Actor.TEAM_ADMIN].user_id], member_user_ids=[ - world.keys[Actor.INTERNAL_USER].user_id, - world.keys[Actor.OWNER].user_id, - world.keys[Actor.UNRELATED_SAME_ORG].user_id, - world.keys[Actor.SERVICE_ACCOUNT].user_id, + actors.keys[Actor.INTERNAL_USER].user_id, + actors.keys[Actor.OWNER].user_id, + actors.keys[Actor.UNRELATED_SAME_ORG].user_id, + actors.keys[Actor.SERVICE_ACCOUNT].user_id, ], ) - return world.org_a_id + return actors.org_a_id if shape == "beta": await create_scratch_team( - prisma, + db, team_id, - organization_id=world.org_b_id, - member_user_ids=[world.keys[Actor.CROSS_ORG_USER].user_id], + organization_id=actors.org_b_id, + member_user_ids=[actors.keys[Actor.CROSS_ORG_USER].user_id], ) - return world.org_b_id + return actors.org_b_id pytest.fail(f"unknown shape={shape}") # pragma: no cover @@ -74,19 +74,19 @@ async def test_team_update_authz_matrix( actor: Actor, shape: str, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - org_id = await _seed_target(prisma, world, shape, scratch.prefix) - caller = world.keys[actor] + org_id = await _seed_target(db, actors, shape, resources.prefix) + caller = actors.keys[actor] - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/update", headers={"Authorization": f"Bearer {caller.cleartext}"}, json={ - "team_id": scratch.prefix, + "team_id": resources.prefix, "team_alias": MARKER_ALIAS, "organization_id": org_id, }, @@ -95,8 +95,8 @@ async def test_team_update_authz_matrix( resp.status_code == expected_status ), f"{actor.value} {shape}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) assert row is not None if expected_status == 200: @@ -106,41 +106,41 @@ async def test_team_update_authz_matrix( async def test_team_update_org_admin_resolved_from_team_without_org_context( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """With no organization_id in the body the route gate resolves the target team's org from team_id, so an org admin of the team's own org is allowed (200), same as PROXY_ADMIN. A team admin of that same team reaches the handler but is refused (403) until a proxy admin enables fields for team admins, and the response says so.""" - await _seed_target(prisma, world, "alpha", scratch.prefix) + await _seed_target(db, actors, "alpha", resources.prefix) - allowed_org_admin = await proxy_client.post( + allowed_org_admin = await gateway_client.post( "/team/update", - headers={"Authorization": f"Bearer {world.keys[Actor.ORG_ADMIN].cleartext}"}, - json={"team_id": scratch.prefix, "team_alias": MARKER_ALIAS}, + headers={"Authorization": f"Bearer {actors.keys[Actor.ORG_ADMIN].cleartext}"}, + json={"team_id": resources.prefix, "team_alias": MARKER_ALIAS}, ) assert allowed_org_admin.status_code == 200, allowed_org_admin.text - allowed_proxy_admin = await proxy_client.post( + allowed_proxy_admin = await gateway_client.post( "/team/update", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - json={"team_id": scratch.prefix, "team_alias": MARKER_ALIAS}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, + json={"team_id": resources.prefix, "team_alias": MARKER_ALIAS}, ) assert allowed_proxy_admin.status_code == 200, allowed_proxy_admin.text - denied_team_admin = await proxy_client.post( + denied_team_admin = await gateway_client.post( "/team/update", - headers={"Authorization": f"Bearer {world.keys[Actor.TEAM_ADMIN].cleartext}"}, - json={"team_id": scratch.prefix, "team_alias": MARKER_ALIAS}, + headers={"Authorization": f"Bearer {actors.keys[Actor.TEAM_ADMIN].cleartext}"}, + json={"team_id": resources.prefix, "team_alias": MARKER_ALIAS}, ) assert denied_team_admin.status_code == 403, denied_team_admin.text assert "cannot edit team settings" in denied_team_admin.text, denied_team_admin.text assert "Team admin editable fields" in denied_team_admin.text, denied_team_admin.text -# Relocation gate — moving a team to a different org. The scratch team starts -# in ORG_A; each scenario relocates it to ORG_B. PROXY_ADMIN bypasses; +# Relocation gate — moving a team to a different org. The resources team starts +# in actors.org_a_id; each scenario relocates it to actors.org_b_id. PROXY_ADMIN bypasses; # ORG_B_ADMIN reaches the handler but holds no role on the source team (403); # ORG_ADMIN holds the source team but not the destination org (403 from the # relocation gate); the team admin is refused by the empty field allow-list and @@ -164,31 +164,31 @@ _RELOCATION = [ async def test_team_update_org_relocation_gate( actor: Actor, expected_status: int, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - await _seed_target(prisma, world, "alpha", scratch.prefix) - caller = world.keys[actor] + await _seed_target(db, actors, "alpha", resources.prefix) + caller = actors.keys[actor] - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/update", headers={"Authorization": f"Bearer {caller.cleartext}"}, - json={"team_id": scratch.prefix, "organization_id": world.org_b_id}, + json={"team_id": resources.prefix, "organization_id": actors.org_b_id}, ) assert ( resp.status_code == expected_status ), f"{actor.value}: {resp.status_code} {resp.text}" - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} + row = await db.db.litellm_teamtable.find_unique( + where={"team_id": resources.prefix} ) assert row is not None if expected_status == 200: - assert row.organization_id == world.org_b_id + assert row.organization_id == actors.org_b_id else: - assert row.organization_id == world.org_a_id, "denied but team relocated" + assert row.organization_id == actors.org_a_id, "denied but team relocated" # Phase 4 F6 — explicit pin on the `team_access_denied` 403 detail string @@ -197,44 +197,44 @@ async def test_team_update_org_relocation_gate( # status; this guard turns a silent rename of the helper's exception detail # into a CI red. async def test_team_update_org_b_admin_relocation_rejection_detail( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): - await _seed_target(prisma, world, "alpha", scratch.prefix) - resp = await proxy_client.post( + await _seed_target(db, actors, "alpha", resources.prefix) + resp = await gateway_client.post( "/team/update", - headers={"Authorization": f"Bearer {world.keys[Actor.ORG_B_ADMIN].cleartext}"}, - json={"team_id": scratch.prefix, "organization_id": world.org_b_id}, + headers={"Authorization": f"Bearer {actors.keys[Actor.ORG_B_ADMIN].cleartext}"}, + json={"team_id": resources.prefix, "organization_id": actors.org_b_id}, ) assert resp.status_code == 403, resp.text assert "do not have access to this team" in resp.text, resp.text async def test_team_update_org_relocation_allowed_for_dual_org_admin( - proxy_client, prisma, scratch, world + gateway_client, db, resources, actors ): """Relocation-allowed branch: a caller who is org admin of BOTH the source and destination org may relocate a team between them. Completes the _RELOCATION matrix, whose allowed branch PR2 left open — no seeded actor is a dual-org admin, so one is minted with create_scratch_actor.""" actor = await create_scratch_actor( - prisma, - scratch.prefix, + db, + resources.prefix, user_role=LitellmUserRoles.ORG_ADMIN.value, - org_admin_of=(world.org_a_id, world.org_b_id), + org_admin_of=(actors.org_a_id, actors.org_b_id), ) team_id = await create_scratch_team( - prisma, scratch.tag("team"), organization_id=world.org_a_id + db, resources.tag("team"), organization_id=actors.org_a_id ) - resp = await proxy_client.post( + resp = await gateway_client.post( "/team/update", headers={"Authorization": f"Bearer {actor.cleartext}"}, - json={"team_id": team_id, "organization_id": world.org_b_id}, + json={"team_id": team_id, "organization_id": actors.org_b_id}, ) assert resp.status_code == 200, resp.text - row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id}) + row = await db.db.litellm_teamtable.find_unique(where={"team_id": team_id}) assert row is not None assert ( - row.organization_id == world.org_b_id + row.organization_id == actors.org_b_id ), "dual-org admin relocation not applied" diff --git a/tests/proxy_behavior/management/test_users_bulk_delete.py b/tests/integration/management/test_users_bulk_delete.py similarity index 61% rename from tests/proxy_behavior/management/test_users_bulk_delete.py rename to tests/integration/management/test_users_bulk_delete.py index 049234e737a..85f19cadffe 100644 --- a/tests/proxy_behavior/management/test_users_bulk_delete.py +++ b/tests/integration/management/test_users_bulk_delete.py @@ -1,7 +1,7 @@ import pytest -from .actors import Actor -from .conftest import create_scratch_team, create_scratch_user +from tests.integration._support.actors import Actor +from tests.integration._support.actors import create_scratch_team, create_scratch_user pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -25,16 +25,16 @@ def _member_ids(row) -> list: return [m["user_id"] for m in (row.members_with_roles or [])] -async def _seed_team_members(prisma, scratch, world, member_ids: list, org_id) -> None: +async def _seed_team_members(db, resources, actors, member_ids: list, org_id) -> None: """Leave behind what /team/member_add would: roster entry, `teams` array, and org membership.""" - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id, member_user_ids=member_ids) - await prisma.db.litellm_usertable.update_many( - where={"user_id": {"in": member_ids}}, data={"teams": {"set": [scratch.prefix]}} + await create_scratch_team(db, resources.prefix, organization_id=actors.org_a_id, member_user_ids=member_ids) + await db.db.litellm_usertable.update_many( + where={"user_id": {"in": member_ids}}, data={"teams": {"set": [resources.prefix]}} ) if org_id is None: return for uid in member_ids: - await prisma.db.litellm_organizationmembership.create( + await db.db.litellm_organizationmembership.create( data={"user_id": uid, "organization_id": org_id, "user_role": "internal_user"} ) @@ -49,25 +49,25 @@ async def test_users_bulk_delete_authz_matrix( org, expected_status: int, expect_deleted: bool, - proxy_client, - prisma, - scratch, - world, + gateway_client, + db, + resources, + actors, ): - victims = [await create_scratch_user(prisma, scratch.prefix, suffix=s) for s in ("v1", "v2")] - keep = await create_scratch_user(prisma, scratch.prefix, suffix="keep") - await _seed_team_members(prisma, scratch, world, victims + [keep], world.org_a_id if org == "a" else None) + victims = [await create_scratch_user(db, resources.prefix, suffix=s) for s in ("v1", "v2")] + keep = await create_scratch_user(db, resources.prefix, suffix="keep") + await _seed_team_members(db, resources, actors, victims + [keep], actors.org_a_id if org == "a" else None) - resp = await proxy_client.post( + resp = await gateway_client.post( _URL, - headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[actor].cleartext}"}, json={"user_ids": victims}, ) assert resp.status_code == expected_status, f"{actor.value}: {resp.status_code} {resp.text}" - team = await prisma.db.litellm_teamtable.find_unique(where={"team_id": scratch.prefix}) + team = await db.db.litellm_teamtable.find_unique(where={"team_id": resources.prefix}) assert team is not None and keep in _member_ids(team), "unrelated member removed" - remaining = {u.user_id for u in await prisma.db.litellm_usertable.find_many(where={"user_id": {"in": victims}})} + remaining = {u.user_id for u in await db.db.litellm_usertable.find_many(where={"user_id": {"in": victims}})} if expected_status == 403: assert resp.headers["content-type"] == "application/problem+json" assert resp.json()["type"] == "urn:litellm:error:forbidden" @@ -79,7 +79,7 @@ async def test_users_bulk_delete_authz_matrix( assert set(body) == {"data"} rows = [(r["user_id"], r["success"], r["teams_removed"]) for r in body["data"]] if expect_deleted: - assert rows == [(v, True, [scratch.prefix]) for v in victims] + assert rows == [(v, True, [resources.prefix]) for v in victims] assert remaining == set() assert not set(victims) & set(_member_ids(team)) return @@ -89,13 +89,13 @@ async def test_users_bulk_delete_authz_matrix( assert set(victims) <= set(_member_ids(team)) -async def test_users_bulk_delete_reports_each_row_in_order(proxy_client, prisma, scratch, world): - victim = await create_scratch_user(prisma, scratch.prefix, suffix="victim") - ghost = scratch.tag("ghost") +async def test_users_bulk_delete_reports_each_row_in_order(gateway_client, db, resources, actors): + victim = await create_scratch_user(db, resources.prefix, suffix="victim") + ghost = resources.tag("ghost") - resp = await proxy_client.post( + resp = await gateway_client.post( _URL, - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"user_ids": [ghost, victim, victim]}, ) assert resp.status_code == 200, resp.text @@ -104,34 +104,34 @@ async def test_users_bulk_delete_reports_each_row_in_order(proxy_client, prisma, (victim, True), (victim, False), ] - assert await prisma.db.litellm_usertable.find_unique(where={"user_id": victim}) is None + assert await db.db.litellm_usertable.find_unique(where={"user_id": victim}) is None -async def test_users_bulk_delete_unknown_query_param_is_400_problem(proxy_client, prisma, scratch, world): - victim = await create_scratch_user(prisma, scratch.prefix, suffix="victim") +async def test_users_bulk_delete_unknown_query_param_is_400_problem(gateway_client, db, resources, actors): + victim = await create_scratch_user(db, resources.prefix, suffix="victim") - resp = await proxy_client.post( + resp = await gateway_client.post( f"{_URL}?dry_run=1", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"user_ids": [victim]}, ) assert resp.status_code == 400, resp.text assert resp.headers["content-type"] == "application/problem+json" assert resp.json()["type"] == "urn:litellm:error:unknown-query-parameter" assert "dry_run" in resp.json()["detail"] - assert await prisma.db.litellm_usertable.find_unique(where={"user_id": victim}) is not None + assert await db.db.litellm_usertable.find_unique(where={"user_id": victim}) is not None -async def test_users_bulk_delete_unknown_body_field_is_422_problem(proxy_client, prisma, scratch, world): - victim = await create_scratch_user(prisma, scratch.prefix, suffix="victim") +async def test_users_bulk_delete_unknown_body_field_is_422_problem(gateway_client, db, resources, actors): + victim = await create_scratch_user(db, resources.prefix, suffix="victim") - resp = await proxy_client.post( + resp = await gateway_client.post( _URL, - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, + headers={"Authorization": f"Bearer {actors.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"user_ids": [victim], "dry_run": True}, ) assert resp.status_code == 422, resp.text assert resp.headers["content-type"] == "application/problem+json" assert resp.json()["type"] == "urn:litellm:error:invalid-request-body" assert "dry_run" in resp.json()["detail"] - assert await prisma.db.litellm_usertable.find_unique(where={"user_id": victim}) is not None + assert await db.db.litellm_usertable.find_unique(where={"user_id": victim}) is not None diff --git a/tests/integration/observability/test_datadog_logging_delivery.py b/tests/integration/observability/test_datadog_logging_delivery.py new file mode 100644 index 00000000000..40853f3e613 --- /dev/null +++ b/tests/integration/observability/test_datadog_logging_delivery.py @@ -0,0 +1,85 @@ +"""Datadog success callback delivers one log payload per completion. + +Deterministic rewrite of the legacy load test: fixed request count, scripted +Datadog intake upstream, asserts the outbound posts arrive rather than a wall +clock overhead ratio. +""" + +import asyncio +import json +import zlib +from typing import Final + +import pytest + +import litellm +from tests._wait_helpers import await_until +from tests.integration._support.wire import Reply, Request, wire_server + +REQUEST_COUNT: Final = 20 + + +def _datadog_upstream(request: Request, captured: list) -> Reply: + assert request.method == "POST" + body = request.body + try: + payload = zlib.decompress(body, zlib.MAX_WBITS | 16) + except zlib.error: + payload = body + captured.append(payload) + return Reply(status=202, body=b"{}") + + +def _delivered_logs(captured: list) -> list: + return [entry for payload in captured for entry in json.loads(payload)] + + +@pytest.mark.asyncio +async def test_datadog_logging_async_delivers_logs(monkeypatch: pytest.MonkeyPatch): + captured: list = [] + + def upstream(request: Request) -> Reply: + return _datadog_upstream(request, captured) + + monkeypatch.setenv("DD_API_KEY", "unit-dd-key") + monkeypatch.setenv("DD_SITE", "us5.datadoghq.com") + + litellm.logging_callback_manager._reset_all_callbacks() + + with wire_server(upstream) as wire: + monkeypatch.setenv("DD_BASE_URL", wire.url) + litellm.success_callback = ["datadog"] + try: + responses = await asyncio.gather( + *( + litellm.acompletion( + model="openai/chatgpt-v-3", + messages=[{"role": "user", "content": "This is a test"}], + max_tokens=5, + user="datadog_latency_test_user", + mock_response="hello from my load test", + ) + for _ in range(REQUEST_COUNT) + ) + ) + assert len(responses) == REQUEST_COUNT + + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=30) + + await await_until( + lambda: len(_delivered_logs(captured)) >= REQUEST_COUNT, + message=f"{REQUEST_COUNT} Datadog log entries delivered", + timeout_s=30, + ) + finally: + litellm.success_callback = [] + litellm.logging_callback_manager._reset_all_callbacks() + + logs = _delivered_logs(captured) + assert len(logs) == REQUEST_COUNT + assert {log["ddsource"] for log in logs} == {"litellm"} + payloads = [json.loads(log["message"]) for log in logs] + assert len({payload["id"] for payload in payloads}) == REQUEST_COUNT + assert {(payload["status"], payload["model"]) for payload in payloads} == {("success", "chatgpt-v-3")} diff --git a/tests/integration/observability/test_langsmith_logging_delivery.py b/tests/integration/observability/test_langsmith_logging_delivery.py new file mode 100644 index 00000000000..8ece3eb9aa2 --- /dev/null +++ b/tests/integration/observability/test_langsmith_logging_delivery.py @@ -0,0 +1,81 @@ +"""Langsmith success callback delivers run payloads for each completion. + +Deterministic rewrite of the legacy load test: fixed request count, scripted +Langsmith upstream, asserts outbound posts arrive rather than a wall clock +overhead ratio. +""" + +import asyncio +import gzip +import json +from typing import Final + +import pytest + +import litellm +from tests._wait_helpers import await_until +from tests.integration._support.wire import Reply, Request, wire_server + +REQUEST_COUNT: Final = 20 + + +def _posted_runs(captured: list) -> list: + return [run for payload in captured for run in json.loads(payload).get("post", [])] + + +@pytest.mark.asyncio +async def test_langsmith_logging_async_delivers_runs(monkeypatch: pytest.MonkeyPatch): + captured: list = [] + + def upstream(request: Request) -> Reply: + assert request.method in ("POST", "PATCH") + body = request.body + try: + payload = gzip.decompress(body) + except OSError: + payload = body + captured.append(payload) + return Reply(status=202, body=b"{}") + + monkeypatch.setenv("LANGSMITH_API_KEY", "lsv2_unit_key") + monkeypatch.setenv("LANGSMITH_PROJECT", "pr-b") + + litellm.logging_callback_manager._reset_all_callbacks() + + with wire_server(upstream) as wire: + monkeypatch.setenv("LANGSMITH_BASE_URL", wire.url) + monkeypatch.setenv("LANGSMITH_ENDPOINT", wire.url) + litellm.success_callback = ["langsmith"] + try: + responses = await asyncio.gather( + *( + litellm.acompletion( + model="openai/chatgpt-v-2", + messages=[{"role": "user", "content": "This is a test"}], + max_tokens=5, + user="langfuse_latency_test_user", + mock_response="hello from my load test", + ) + for _ in range(REQUEST_COUNT) + ) + ) + assert len(responses) == REQUEST_COUNT + + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=30) + + await await_until( + lambda: len(_posted_runs(captured)) >= REQUEST_COUNT, + message=f"{REQUEST_COUNT} LangSmith runs posted", + timeout_s=30, + ) + finally: + litellm.success_callback = [] + litellm.logging_callback_manager._reset_all_callbacks() + + runs = _posted_runs(captured) + assert len(runs) == REQUEST_COUNT + assert {(run["run_type"], run["session_name"]) for run in runs} == {("llm", "pr-b")} + assert len({run["inputs"]["id"] for run in runs}) == REQUEST_COUNT + assert {run["inputs"]["model"] for run in runs} == {"chatgpt-v-2"} diff --git a/tests/integration/observability/test_passthrough_upstream_error_visibility.py b/tests/integration/observability/test_passthrough_upstream_error_visibility.py index c29ce4f1c62..bc6747844ba 100644 --- a/tests/integration/observability/test_passthrough_upstream_error_visibility.py +++ b/tests/integration/observability/test_passthrough_upstream_error_visibility.py @@ -223,6 +223,57 @@ def test_gemini_passthrough_streaming_200_relays_every_chunk(gateway: Gateway, t assert not _upstream_warnings(owned.log), owned.log.read_text()[-2000:] +def test_gemini_streaming_passthrough_request_tags_reach_spend_log(gateway: Gateway, tmp_path: Path) -> None: + expected_body: Final = {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]} + chunks: Final = ( + b'data: {"candidates":[{"content":{"parts":[{"text":"hello"}],"role":"model"},"index":0}]}\n\n', + b'data: {"candidates":[{"finishReason":"STOP","index":0}],"usageMetadata":{"promptTokenCount":1,"candidatesTokenCount":1,"totalTokenCount":2}}\n\n', + ) + + def respond(request: Request) -> Reply: + assert request.method == "POST", request + assert request.target.split("?", 1)[0] == "/v1beta/models/claude-nope-9:streamGenerateContent", request + assert json.loads(request.body) == expected_body, request + return Reply(status=200, chunks=chunks, content_type="text/event-stream") + + config_path: Final = tmp_path / "gemini-stream-tags.yaml" + with wire_server(respond) as wire: + _gemini_config(config_path, wire.url) + with owned_proxy_process(gateway, tmp_path, {}, config=config_path, workers=2) as owned: + candidate: Final = owned.gateway + with candidate.client.stream( + "POST", + _GEMINI_STREAM_PATH, + params={"alt": "sse"}, + json=expected_body, + headers={**_gemini_headers(candidate), "tags": "gemini-js-sdk,pass-through-endpoint"}, + ) as response: + assert response.status_code == 200 + streamed: Final = response.read() + call_id: Final = response.headers["x-litellm-call-id"] + assert streamed == b"".join(chunks) + + rows: Final = eventually( + lambda: read_rows( + 'SELECT request_tags, call_type, status FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (call_id,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + row: Final = rows[0] + raw_tags: Final = row["request_tags"] + request_tags: Final = json.loads(raw_tags) if isinstance(raw_tags, str) else raw_tags + policy_tags: Final = [ + tag + for tag in request_tags + if not (isinstance(tag, str) and tag.startswith("User-Agent: ")) + ] + assert policy_tags == ["gemini-js-sdk", "pass-through-endpoint"], row + assert row["call_type"] == "pass_through_endpoint", row + assert row["status"] == "success", row + + def test_config_pass_through_route_logs_body_and_strips_query(gateway: Gateway, tmp_path: Path) -> None: upstream_error: Final = {"error": {"message": "max budget reached for this deployment"}} diff --git a/tests/integration/providers/test_a2a_sdk_upstream_wire.py b/tests/integration/providers/test_a2a_sdk_upstream_wire.py new file mode 100644 index 00000000000..098d1b742b5 --- /dev/null +++ b/tests/integration/providers/test_a2a_sdk_upstream_wire.py @@ -0,0 +1,335 @@ +"""A2A SDK paths against a scripted upstream agent. + +Ports the legacy agent_tests coverage: asend_message with an explicit A2A +client, the LiteLLM logging payload for A2A calls, the pydantic_ai_agents +provider's send-and-poll flow, and the completion bridge driving LangGraph +/runs/wait and /runs/stream endpoints. +""" + +import asyncio +import json +from typing import Final +from uuid import uuid4 + +import pytest +from a2a.compat.v0_3.types import ( + MessageSendParams, + SendMessageRequest, + SendStreamingMessageRequest, +) + +import litellm +from tests.integration._support.wire import Reply, Request, wire_server + + +def _agent_card(wire_url: str) -> dict: + return { + "protocolVersion": "0.3", + "name": "scripted-a2a-agent", + "description": "scripted upstream", + "version": "1.0.0", + "url": wire_url + "/", + "capabilities": {"streaming": True}, + "defaultInputModes": ["text"], + "defaultOutputModes": ["text"], + "skills": [], + } + + +def _a2a_message_response(request: Request, text: str) -> Reply: + body = json.loads(request.body) + return Reply( + body=json.dumps( + { + "jsonrpc": "2.0", + "id": body["id"], + "result": { + "kind": "message", + "role": "agent", + "messageId": uuid4().hex, + "parts": [{"kind": "text", "text": text}], + }, + } + ).encode() + ) + + +def _send_request() -> SendMessageRequest: + send_message_payload = { + "message": { + "role": "user", + "parts": [{"kind": "text", "text": "Hello from the test"}], + "messageId": uuid4().hex, + }, + } + return SendMessageRequest(id=str(uuid4()), params=MessageSendParams(**send_message_payload)) + + +@pytest.mark.asyncio +async def test_asend_message_with_client_decorator(): + """asend_message over an explicit A2A client returns the agent message.""" + from litellm.a2a_protocol import asend_message, create_a2a_client + + def upstream(request: Request) -> Reply: + if request.method == "GET": + assert request.target in ("/.well-known/agent-card.json", "/.well-known/agent.json") + return Reply(body=json.dumps(_agent_card(wire.url)).encode()) + return _a2a_message_response(request, "scripted agent reply") + + with wire_server(upstream) as wire: + a2a_client = await create_a2a_client(base_url=wire.url) + response = await asend_message(a2a_client=a2a_client, request=_send_request()) + + assert response is not None + result = response.result if hasattr(response, "result") else response + result_dict = result if isinstance(result, dict) else result.model_dump(mode="python", exclude_none=True) + assert result_dict["kind"] == "message" + assert result_dict["role"] == "agent" + assert result_dict["parts"][0]["text"] == "scripted agent reply" + + +@pytest.mark.asyncio +async def test_a2a_logging_payload(): + """A2A calls produce a standard logging payload with provider and zero cost.""" + from litellm.integrations.custom_logger import CustomLogger + + class TestA2ALogger(CustomLogger): + def __init__(self): + self.standard_logging_payload = None + self.log_success_called = False + super().__init__() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.log_success_called = True + self.standard_logging_payload = kwargs.get("standard_logging_object", None) + + litellm.logging_callback_manager._reset_all_callbacks() + test_logger = TestA2ALogger() + litellm.callbacks = [test_logger] + + def upstream(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=json.dumps(_agent_card(wire.url)).encode()) + return _a2a_message_response(request, "scripted agent reply") + + try: + from litellm.a2a_protocol import asend_message, create_a2a_client + + with wire_server(upstream) as wire: + a2a_client = await create_a2a_client(base_url=wire.url) + await asend_message(a2a_client=a2a_client, request=_send_request()) + + deadline = asyncio.get_event_loop().time() + 10 + while not test_logger.log_success_called and asyncio.get_event_loop().time() < deadline: + await asyncio.sleep(0.05) + finally: + litellm.logging_callback_manager._reset_all_callbacks() + + assert test_logger.log_success_called is True + slp = test_logger.standard_logging_payload + assert slp is not None + + logged_model = slp.get("model") if isinstance(slp, dict) else getattr(slp, "model", None) + logged_provider = ( + slp.get("custom_llm_provider") if isinstance(slp, dict) else getattr(slp, "custom_llm_provider", None) + ) + call_type = slp.get("call_type") if isinstance(slp, dict) else getattr(slp, "call_type", None) + response_cost = ( + slp.get("response_cost") if isinstance(slp, dict) else getattr(slp, "response_cost", None) + ) + + assert logged_model is not None and "a2a_agent/" in logged_model + assert logged_provider == "a2a_agent" + assert call_type == "asend_message" + assert response_cost == 0.0 + + +def _pydantic_task_response(request: Request) -> Reply: + body = json.loads(request.body) + assert body["method"] in ("message/send", "tasks/get"), body["method"] + return Reply( + body=json.dumps( + { + "jsonrpc": "2.0", + "id": body["id"], + "result": { + "id": "task-1", + "kind": "task", + "status": {"state": "completed"}, + "history": [ + body["params"]["message"] if body["method"] == "message/send" else {}, + { + "role": "agent", + "kind": "message", + "messageId": "agent-msg-1", + "parts": [{"kind": "text", "text": "pydantic agent reply"}], + }, + ], + "artifacts": [ + { + "artifactId": "artifact-1", + "parts": [{"kind": "text", "text": "pydantic agent reply"}], + } + ], + }, + } + ).encode() + ) + + +@pytest.mark.asyncio +async def test_pydantic_ai_non_streaming(): + """pydantic_ai_agents send returns the completed task with agent text.""" + from litellm.a2a_protocol import asend_message + + with wire_server(_pydantic_task_response) as wire: + response = await asend_message( + request=_send_request(), + api_base=wire.url, + litellm_params={"custom_llm_provider": "pydantic_ai_agents"}, + ) + + assert response is not None + result = response.result + assert result is not None + result_dict = result if isinstance(result, dict) else result.model_dump(mode="python", exclude_none=True) + assert result_dict["kind"] == "message" + assert result_dict["role"] == "agent" + assert result_dict["parts"][0]["kind"] == "text" + assert result_dict["parts"][0]["text"] == "pydantic agent reply" + + +@pytest.mark.asyncio +async def test_pydantic_ai_fake_streaming(): + """pydantic_ai_agents fake streaming emits task, working, artifact, completed.""" + from litellm.a2a_protocol import asend_message_streaming + + send_message_payload = { + "message": { + "role": "user", + "parts": [{"kind": "text", "text": "Hello from Pydantic AI streaming test!"}], + "messageId": uuid4().hex, + }, + } + request = SendStreamingMessageRequest(id=str(uuid4()), params=MessageSendParams(**send_message_payload)) + + with wire_server(_pydantic_task_response) as wire: + chunks = [] + async for chunk in asend_message_streaming( + request=request, + api_base=wire.url, + litellm_params={"custom_llm_provider": "pydantic_ai_agents"}, + ): + chunk_dict = chunk.model_dump(mode="json", exclude_none=True) if hasattr(chunk, "model_dump") else chunk + chunks.append(chunk_dict) + + kinds = [c.get("result", {}).get("kind") for c in chunks] + states = [c.get("result", {}).get("status", {}).get("state") for c in chunks] + assert len(chunks) > 0 + assert "task" in kinds + assert "artifact-update" in kinds + assert "working" in states + assert "completed" in states + + +def _langgraph_upstream(request: Request) -> Reply: + assert request.method == "POST" + body = json.loads(request.body) + messages = body["input"]["messages"] + assert messages[0]["role"] == "human" + if request.target == "/runs/wait": + return Reply( + body=json.dumps( + {"messages": [{"type": "ai", "role": "assistant", "content": "2 + 2 = 4."}]} + ).encode() + ) + assert request.target == "/runs/stream" + sse = ( + 'data: {"content":"1, "}\n\n' + 'data: {"content":"2, 3, 4, 5."}\n\n' + ) + return Reply(body=sse.encode(), content_type="text/event-stream") + + +@pytest.mark.asyncio +async def test_a2a_completion_bridge_non_streaming(): + """Completion bridge non-streaming returns the LangGraph text as an A2A message.""" + from litellm.a2a_protocol import asend_message + + send_message_payload = { + "message": { + "role": "user", + "parts": [{"kind": "text", "text": "What is 2 + 2?"}], + "messageId": uuid4().hex, + }, + } + request = SendMessageRequest(id=str(uuid4()), params=MessageSendParams(**send_message_payload)) + + with wire_server(_langgraph_upstream) as wire: + response = await asend_message( + request=request, + api_base=wire.url, + litellm_params={"custom_llm_provider": "langgraph", "model": "agent"}, + ) + + assert response.jsonrpc == "2.0" + assert response.id is not None + assert response.result is not None + assert response.result.get("kind") == "message" + + message = response.result + assert message["role"] == "agent" + assert len(message["parts"]) > 0 + assert message["parts"][0]["kind"] == "text" + assert message["parts"][0]["text"] == "2 + 2 = 4." + + +@pytest.mark.asyncio +async def test_a2a_completion_bridge_streaming(): + """Completion bridge streaming emits task, working, artifact, completed events.""" + from litellm.a2a_protocol import asend_message_streaming + + send_message_payload = { + "message": { + "role": "user", + "parts": [{"kind": "text", "text": "Count from 1 to 5."}], + "messageId": uuid4().hex, + }, + } + request = SendStreamingMessageRequest(id=str(uuid4()), params=MessageSendParams(**send_message_payload)) + + with wire_server(_langgraph_upstream) as wire: + chunks = [] + async for chunk in asend_message_streaming( + request=request, + api_base=wire.url, + litellm_params={"custom_llm_provider": "langgraph", "model": "agent"}, + ): + chunks.append(chunk) + + assert len(chunks) >= 4, f"Expected at least 4 chunks, got {len(chunks)}" + + for chunk in chunks: + assert chunk["jsonrpc"] == "2.0" + assert "id" in chunk + assert "result" in chunk + + task_chunk = chunks[0] + assert task_chunk["result"]["kind"] == "task" + assert task_chunk["result"]["status"]["state"] == "submitted" + + working_chunk = chunks[1] + assert working_chunk["result"]["kind"] == "status-update" + assert working_chunk["result"]["status"]["state"] == "working" + assert working_chunk["result"]["final"] is False + + artifact_chunk = chunks[2] + assert artifact_chunk["result"]["kind"] == "artifact-update" + parts = artifact_chunk["result"]["artifact"]["parts"] + assert len(parts) > 0 + assert parts[0]["kind"] == "text" + + final_chunk = chunks[-1] + assert final_chunk["result"]["kind"] == "status-update" + assert final_chunk["result"]["status"]["state"] == "completed" + assert final_chunk["result"]["final"] is True diff --git a/tests/proxy_behavior/management/__init__.py b/tests/integration/spend/__init__.py similarity index 100% rename from tests/proxy_behavior/management/__init__.py rename to tests/integration/spend/__init__.py diff --git a/tests/proxy_behavior/spend/test_autorouter_session_rollup.py b/tests/integration/spend/test_autorouter_session_rollup.py similarity index 87% rename from tests/proxy_behavior/spend/test_autorouter_session_rollup.py rename to tests/integration/spend/test_autorouter_session_rollup.py index 3425bed22b2..a3b959040b5 100644 --- a/tests/proxy_behavior/spend/test_autorouter_session_rollup.py +++ b/tests/integration/spend/test_autorouter_session_rollup.py @@ -6,12 +6,18 @@ tests/unit/proxy/db/test_autorouter_session_rollup.py. """ import asyncio +import hashlib import time import uuid +from collections.abc import AsyncIterator, Iterator +from dataclasses import dataclass +from contextvars import ContextVar, Token from datetime import datetime, timedelta, timezone +from itertools import count from types import SimpleNamespace from typing import Final, TypedDict, cast +import pytest_asyncio import pytest from prisma import Prisma from prisma.errors import RawQueryError @@ -26,12 +32,49 @@ from litellm.proxy.db.autorouter_session_rollup import ( write_autorouter_turn, ) from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import SpendLogCleanup +from tests.integration._support.prisma_clients import connected_prisma pytestmark = pytest.mark.asyncio(loop_scope="session") T0 = datetime(2026, 8, 1, 12, 0, 0) +@dataclass(frozen=True, slots=True) +class ScopedIdentity: + namespace: str + sequence: Iterator[int] + + def new(self) -> str: + return f"{self.namespace}-{next(self.sequence)}" + + +IDENTITY: Final = ContextVar[ScopedIdentity | None]("autorouter_test_identity", default=None) + + +def _new_identity() -> str: + identity: Final = IDENTITY.get() + assert identity is not None + return identity.new() + + +@pytest_asyncio.fixture(scope="module", loop_scope="session") +async def db() -> AsyncIterator[Prisma]: + async with connected_prisma() as client: + yield client + + +@pytest_asyncio.fixture(autouse=True, loop_scope="session") +async def cleanup_rows(db: Prisma) -> AsyncIterator[None]: + identity: Final = ScopedIdentity(uuid.uuid4().hex, count()) + token: Final[Token[ScopedIdentity | None]] = IDENTITY.set(identity) + yield + key_pattern: Final = f"k-{identity.namespace}-%" + await db.execute_raw('DELETE FROM "LiteLLM_AutoRouterDailySpend" WHERE api_key LIKE $1', key_pattern) + await db.execute_raw('DELETE FROM "LiteLLM_AutoRouterUserSession" WHERE api_key LIKE $1', key_pattern) + await db.execute_raw('DELETE FROM "LiteLLM_AutoRouterSession" WHERE api_key LIKE $1', key_pattern) + IDENTITY.reset(token) + + def _utc_epoch(moment: datetime) -> float: return moment.replace(tzinfo=timezone.utc).timestamp() @@ -115,7 +158,7 @@ async def _row(db, key: str, session_id: str = "s1", router: str = "auto-1") -> async def test_every_turn_lands_in_exactly_one_bucket(db): - key = f"k-{uuid.uuid4()}" + key = f"k-{_new_identity()}" await _turn(db, key, "A", T0, ttl=300) await _turn(db, key, "A", T0 + timedelta(seconds=10), hit=1) await _turn(db, key, "B", T0 + timedelta(seconds=20), ttl=3600) @@ -145,7 +188,7 @@ async def test_every_turn_lands_in_exactly_one_bucket(db): async def test_return_misses_attribute_against_the_recorded_ttl(db): - key = f"k-{uuid.uuid4()}" + key = f"k-{_new_identity()}" await _turn(db, key, "A", T0, ttl=300) await _turn(db, key, "B", T0 + timedelta(seconds=10), ttl=3600) await _turn(db, key, "A", T0 + timedelta(seconds=400)) @@ -157,7 +200,7 @@ async def test_return_misses_attribute_against_the_recorded_ttl(db): async def test_a_return_miss_with_no_recorded_ttl_stays_unattributed(db): - key = f"k-{uuid.uuid4()}" + key = f"k-{_new_identity()}" await _turn(db, key, "A", T0) await _turn(db, key, "B", T0 + timedelta(seconds=10)) await _turn(db, key, "A", T0 + timedelta(seconds=20)) @@ -169,7 +212,7 @@ async def test_a_return_miss_with_no_recorded_ttl_stays_unattributed(db): async def test_a_hit_refreshes_the_models_cache_clock(db): - key = f"k-{uuid.uuid4()}" + key = f"k-{_new_identity()}" await _turn(db, key, "A", T0, ttl=300) await _turn(db, key, "B", T0 + timedelta(seconds=250), ttl=3600) await _turn(db, key, "A", T0 + timedelta(seconds=290), hit=1) @@ -183,7 +226,7 @@ async def test_a_hit_refreshes_the_models_cache_clock(db): async def test_out_of_order_turns_do_not_rewind_the_session(db): - key = f"k-{uuid.uuid4()}" + key = f"k-{_new_identity()}" await _turn(db, key, "A", T0 + timedelta(seconds=100), ttl=300) await _turn(db, key, "B", T0 + timedelta(seconds=200)) await _turn(db, key, "A", T0) @@ -197,7 +240,7 @@ async def test_out_of_order_turns_do_not_rewind_the_session(db): async def test_concurrent_writers_compose_without_losing_turns(db): - key = f"k-{uuid.uuid4()}" + key = f"k-{_new_identity()}" await _turn(db, key, "A", T0, classifier_cost=0.001) await asyncio.gather( *(_turn(db, key, "A", T0 + timedelta(seconds=1 + offset), hit=1, classifier_cost=0.002) for offset in range(30)) @@ -231,7 +274,7 @@ async def _legacy_turn(db, key: str, at: datetime, session_id: str = "s1", route @pytest.mark.parametrize("writers", [(False,), (True,), (False, True), (True, False)]) async def test_subtotal_coverage_survives_legacy_and_rolling_writers(db, writers: tuple[bool, ...]): - key: Final = f"k-{uuid.uuid4()}" + key: Final = f"k-{_new_identity()}" for offset, records_cost in enumerate(writers): at: Final = T0 + timedelta(seconds=offset) if records_cost: @@ -260,7 +303,7 @@ async def test_subtotal_coverage_survives_legacy_and_rolling_writers(db, writers async def test_unknown_and_legacy_turns_preserve_actual_spend_without_entering_the_estimated_cohort(db: Prisma) -> None: - key: Final = f"k-{uuid.uuid4()}" + key: Final = f"k-{_new_identity()}" await _turn(db, key, "A", T0, spend=0.25, saved=-0.05, baseline="opus") await _turn( db, key, "B", T0 + timedelta(seconds=1), spend=0.7, saved=0, baseline="sonnet", estimated=False @@ -280,10 +323,10 @@ async def test_unknown_and_legacy_turns_preserve_actual_spend_without_entering_t async def test_the_benchmarks_aggregate_reads_only_overlapping_sessions(db): - key = f"k-{uuid.uuid4()}" - router = f"r-{uuid.uuid4()}" - in_window = f"s-{uuid.uuid4()}" - out_of_window = f"s-{uuid.uuid4()}" + key = f"k-{_new_identity()}" + router = f"r-{_new_identity()}" + in_window = f"s-{_new_identity()}" + out_of_window = f"s-{_new_identity()}" await _turn( db, key, @@ -315,9 +358,9 @@ async def test_the_benchmarks_aggregate_reads_only_overlapping_sessions(db): async def test_the_benchmarks_aggregate_can_filter_to_one_key(db): - router = f"r-{uuid.uuid4()}" - first_key = f"k-{uuid.uuid4()}" - second_key = f"k-{uuid.uuid4()}" + router = f"r-{_new_identity()}" + first_key = f"k-{_new_identity()}" + second_key = f"k-{_new_identity()}" await _turn(db, first_key, "A", T0, router=router, saved=0.5, classifier_cost=0.01) await _turn(db, second_key, "A", T0, router=router, saved=9.0, classifier_cost=0.09) @@ -330,7 +373,7 @@ async def test_the_benchmarks_aggregate_can_filter_to_one_key(db): assert day["classifier_cost"] == pytest.approx(0.01) assert day["classifier_cost_recorded_turns"] == 1 - unknown_key_rows = await _benchmark_rows(db, (T0 - timedelta(days=1)), (T0 + timedelta(days=1)), f"k-{uuid.uuid4()}", None) + unknown_key_rows = await _benchmark_rows(db, (T0 - timedelta(days=1)), (T0 + timedelta(days=1)), f"k-{_new_identity()}", None) assert [row for row in unknown_key_rows if row["router_name"] == router] == [] @@ -360,11 +403,11 @@ async def _scoped_benchmarks( async def test_users_keep_written_identity_across_shared_keys_and_keyless_sessions(db: Prisma) -> None: - router: Final = f"r-{uuid.uuid4()}" - alice: Final = f"u-{uuid.uuid4()}" - bob: Final = f"u-{uuid.uuid4()}" - first_key: Final = f"k-{uuid.uuid4()}" - second_key: Final = f"k-{uuid.uuid4()}" + router: Final = f"r-{_new_identity()}" + alice: Final = f"u-{_new_identity()}" + bob: Final = f"u-{_new_identity()}" + first_key: Final = f"k-{_new_identity()}" + second_key: Final = f"k-{_new_identity()}" await _legacy_turn(db, first_key, T0, router=router) await _turn(db, first_key, "A", T0 + timedelta(seconds=10), router=router, user_id=alice, tier="simple") await _turn( @@ -404,7 +447,7 @@ async def test_users_keep_written_identity_across_shared_keys_and_keyless_sessio assert (intersection[0]["sessions"], intersection[0]["turns"]) == (1, 1) assert intersection[0]["spend"] == pytest.approx(0.01) assert await _scoped_benchmarks(db, router, user_id=bob, key=second_key) == () - assert await _scoped_benchmarks(db, router, user_id=f"u-{uuid.uuid4()}") == () + assert await _scoped_benchmarks(db, router, user_id=f"u-{_new_identity()}") == () assert [ row for row in await _benchmark_rows(db, (T0 - timedelta(days=1)), (T0 + timedelta(days=1)), None, "") @@ -413,8 +456,8 @@ async def test_users_keep_written_identity_across_shared_keys_and_keyless_sessio async def test_a_failed_user_projection_rolls_back_the_keys_increment(db: Prisma) -> None: - key: Final = f"k-{uuid.uuid4()}" - user_id: Final = "".join(str(uuid.uuid4()) for _ in range(200)) + key: Final = f"k-{_new_identity()}" + user_id: Final = hashlib.shake_256(_new_identity().encode()).hexdigest(4096) await _turn(db, key, "A", T0) before: Final = await _row(db, key) @@ -425,8 +468,8 @@ async def test_a_failed_user_projection_rolls_back_the_keys_increment(db: Prisma assert await db.query_raw('SELECT user_id FROM "LiteLLM_AutoRouterUserSession" WHERE user_id = $1', user_id) == [] assert [day["turns"] for day in await _days(db, key)] == [1] - first_user: Final = f"u-{uuid.uuid4()}" - second_user: Final = f"u-{uuid.uuid4()}" + first_user: Final = f"u-{_new_identity()}" + second_user: Final = f"u-{_new_identity()}" turns: Final = tuple( AutoRouterTurnTransaction( api_key=key, @@ -479,9 +522,9 @@ async def test_a_failed_user_projection_rolls_back_the_keys_increment(db: Prisma async def test_user_session_cleanup_keeps_another_users_recent_keyless_session(db: Prisma) -> None: - router: Final = f"r-{uuid.uuid4()}" - expired_user: Final = f"u-{uuid.uuid4()}" - recent_user: Final = f"u-{uuid.uuid4()}" + router: Final = f"r-{_new_identity()}" + expired_user: Final = f"u-{_new_identity()}" + recent_user: Final = f"u-{_new_identity()}" await _turn(db, "", "A", T0 - timedelta(days=1), router=router, user_id=expired_user) await _turn(db, "", "A", T0 + timedelta(days=1), router=router, user_id=recent_user) cleaner: Final = SpendLogCleanup(general_settings={}) @@ -496,11 +539,11 @@ async def test_user_session_cleanup_keeps_another_users_recent_keyless_session(d async def test_a_reconfigured_alias_reports_each_router_type_as_its_own_group(db): - key = f"k-{uuid.uuid4()}" - router = f"r-{uuid.uuid4()}" - await _turn(db, key, "A", T0, session_id=f"s-{uuid.uuid4()}", router=router, router_type="complexity") + key = f"k-{_new_identity()}" + router = f"r-{_new_identity()}" + await _turn(db, key, "A", T0, session_id=f"s-{_new_identity()}", router=router, router_type="complexity") await _turn( - db, key, "A", T0 + timedelta(seconds=10), session_id=f"s-{uuid.uuid4()}", router=router, router_type="quality" + db, key, "A", T0 + timedelta(seconds=10), session_id=f"s-{_new_identity()}", router=router, router_type="quality" ) rows = await _benchmark_rows(db, (T0 - timedelta(days=1)), (T0 + timedelta(days=1)), None, None) @@ -512,7 +555,7 @@ async def test_a_reconfigured_alias_reports_each_router_type_as_its_own_group(db async def test_tier_turns_count_each_tier_that_served_a_turn(db): - key = f"k-{uuid.uuid4()}" + key = f"k-{_new_identity()}" await _turn(db, key, "A", T0, tier="simple") await _turn(db, key, "B", T0 + timedelta(seconds=10), tier="complex") await _turn(db, key, "A", T0 + timedelta(seconds=20), tier="simple") @@ -521,7 +564,7 @@ async def test_tier_turns_count_each_tier_that_served_a_turn(db): async def test_an_untiered_turn_increments_no_tier_counter(db): - key = f"k-{uuid.uuid4()}" + key = f"k-{_new_identity()}" await _turn(db, key, "A", T0, tier=None) assert (await _row(db, key))["tier_turns"] == {} @@ -533,7 +576,7 @@ async def test_an_untiered_turn_increments_no_tier_counter(db): async def test_a_mid_session_router_type_change_keeps_foreign_tier_names_out_of_the_map(db): - key = f"k-{uuid.uuid4()}" + key = f"k-{_new_identity()}" await _turn(db, key, "A", T0, router_type="complexity", tier="medium") await _turn(db, key, "A", T0 + timedelta(seconds=10), router_type="quality", tier="2") await _turn(db, key, "A", T0 + timedelta(seconds=20), router_type="complexity", tier="medium") @@ -544,7 +587,7 @@ async def test_a_mid_session_router_type_change_keeps_foreign_tier_names_out_of_ async def test_baseline_models_count_the_turns_priced_against_each_baseline(db): - key = f"k-{uuid.uuid4()}" + key = f"k-{_new_identity()}" await _turn(db, key, "A", T0, baseline="opus") await _turn(db, key, "B", T0 + timedelta(seconds=10), baseline="opus") await _turn(db, key, "A", T0 + timedelta(seconds=20), baseline="sonnet") @@ -553,7 +596,7 @@ async def test_baseline_models_count_the_turns_priced_against_each_baseline(db): async def test_a_turn_priced_against_no_baseline_leaves_the_map_alone(db): - key = f"k-{uuid.uuid4()}" + key = f"k-{_new_identity()}" await _turn(db, key, "A", T0, baseline=None) assert (await _row(db, key))["baseline_models"] == {} @@ -565,7 +608,7 @@ async def test_a_turn_priced_against_no_baseline_leaves_the_map_alone(db): async def test_an_out_of_order_turn_still_counts_toward_its_tier(db): - key = f"k-{uuid.uuid4()}" + key = f"k-{_new_identity()}" await _turn(db, key, "A", T0 + timedelta(seconds=60), tier="simple") await _turn(db, key, "A", T0, tier="simple") @@ -575,12 +618,12 @@ async def test_an_out_of_order_turn_still_counts_toward_its_tier(db): async def test_the_benchmarks_aggregate_sums_tier_turns_across_sessions(db): - key = f"k-{uuid.uuid4()}" - router = f"r-{uuid.uuid4()}" - await _turn(db, key, "A", T0, session_id=f"s-{uuid.uuid4()}", router=router, tier="simple") - await _turn(db, key, "A", T0 + timedelta(seconds=10), session_id=f"s-{uuid.uuid4()}", router=router, tier="simple") - await _turn(db, key, "B", T0 + timedelta(seconds=20), session_id=f"s-{uuid.uuid4()}", router=router, tier="complex") - await _turn(db, key, "C", T0 + timedelta(seconds=30), session_id=f"s-{uuid.uuid4()}", router=router, tier=None) + key = f"k-{_new_identity()}" + router = f"r-{_new_identity()}" + await _turn(db, key, "A", T0, session_id=f"s-{_new_identity()}", router=router, tier="simple") + await _turn(db, key, "A", T0 + timedelta(seconds=10), session_id=f"s-{_new_identity()}", router=router, tier="simple") + await _turn(db, key, "B", T0 + timedelta(seconds=20), session_id=f"s-{_new_identity()}", router=router, tier="complex") + await _turn(db, key, "C", T0 + timedelta(seconds=30), session_id=f"s-{_new_identity()}", router=router, tier=None) rows = await _benchmark_rows(db, (T0 - timedelta(days=1)), (T0 + timedelta(days=1)), None, None) grouped = next(row for row in rows if row["router_name"] == router) @@ -589,17 +632,17 @@ async def test_the_benchmarks_aggregate_sums_tier_turns_across_sessions(db): async def test_tier_maps_stay_separate_per_router_type_on_a_reconfigured_alias(db): - key = f"k-{uuid.uuid4()}" - router = f"r-{uuid.uuid4()}" + key = f"k-{_new_identity()}" + router = f"r-{_new_identity()}" await _turn( - db, key, "A", T0, session_id=f"s-{uuid.uuid4()}", router=router, router_type="complexity", tier="medium" + db, key, "A", T0, session_id=f"s-{_new_identity()}", router=router, router_type="complexity", tier="medium" ) await _turn( db, key, "A", T0 + timedelta(seconds=10), - session_id=f"s-{uuid.uuid4()}", + session_id=f"s-{_new_identity()}", router=router, router_type="quality", tier="2", @@ -611,9 +654,9 @@ async def test_tier_maps_stay_separate_per_router_type_on_a_reconfigured_alias(d async def test_a_window_with_no_tiered_turns_aggregates_to_an_empty_map(db): - key = f"k-{uuid.uuid4()}" - router = f"r-{uuid.uuid4()}" - await _turn(db, key, "A", T0, session_id=f"s-{uuid.uuid4()}", router=router, tier=None) + key = f"k-{_new_identity()}" + router = f"r-{_new_identity()}" + await _turn(db, key, "A", T0, session_id=f"s-{_new_identity()}", router=router, tier=None) rows = await _benchmark_rows(db, (T0 - timedelta(days=1)), (T0 + timedelta(days=1)), None, None) grouped = next(row for row in rows if row["router_name"] == router) @@ -621,7 +664,7 @@ async def test_a_window_with_no_tiered_turns_aggregates_to_an_empty_map(db): async def test_a_miss_that_touched_no_cache_does_not_advance_the_ttl_clock(db): - key = f"k-{uuid.uuid4()}" + key = f"k-{_new_identity()}" await _turn(db, key, "A", T0, ttl=300) await _turn(db, key, "B", T0 + timedelta(seconds=10), ttl=3600) await _turn(db, key, "A", T0 + timedelta(seconds=400)) @@ -634,7 +677,7 @@ async def test_a_miss_that_touched_no_cache_does_not_advance_the_ttl_clock(db): async def test_an_out_of_order_hit_still_counts_toward_the_overall_hit_rate(db): - key = f"k-{uuid.uuid4()}" + key = f"k-{_new_identity()}" await _turn(db, key, "A", T0 + timedelta(seconds=100)) await _turn(db, key, "A", T0 + timedelta(seconds=50), hit=1) @@ -645,8 +688,8 @@ async def test_an_out_of_order_hit_still_counts_toward_the_overall_hit_rate(db): async def test_a_cross_midnight_session_splits_its_money_by_request_day(db): - key = f"k-{uuid.uuid4()}" - router = f"auto-{uuid.uuid4()}" + key = f"k-{_new_identity()}" + router = f"auto-{_new_identity()}" midnight = datetime(2026, 9, 2) await _turn(db, key, "A", midnight - timedelta(minutes=10), router=router, tokens=100, spend=1.0, saved=7.0, user_id="u1") await _turn(db, key, "A", midnight + timedelta(minutes=10), router=router, tokens=200, spend=1.0, saved=3.0, user_id="u1") @@ -669,8 +712,8 @@ async def test_a_cross_midnight_session_splits_its_money_by_request_day(db): async def test_a_router_type_change_within_a_day_keeps_each_types_money_apart(db): - key = f"k-{uuid.uuid4()}" - router = f"auto-{uuid.uuid4()}" + key = f"k-{_new_identity()}" + router = f"auto-{_new_identity()}" await _turn(db, key, "A", T0, router=router, router_type="complexity", spend=1.0, saved=4.0) await _turn(db, key, "A", T0 + timedelta(hours=1), router=router, router_type="quality", spend=2.0, saved=0.0) @@ -679,8 +722,8 @@ async def test_a_router_type_change_within_a_day_keeps_each_types_money_apart(db async def test_a_router_type_change_mid_session_keeps_session_shape_with_the_sessions_type(db): - key = f"k-{uuid.uuid4()}" - router = f"auto-{uuid.uuid4()}" + key = f"k-{_new_identity()}" + router = f"auto-{_new_identity()}" await _turn(db, key, "A", T0, router=router, router_type="complexity", spend=1.0, saved=4.0) await _turn(db, key, "B", T0 + timedelta(hours=1), router=router, router_type="quality", spend=2.0, saved=0.0) @@ -693,8 +736,8 @@ async def test_a_router_type_change_mid_session_keeps_session_shape_with_the_ses @pytest.mark.parametrize("statement", [UPSERT_AUTOROUTER_SESSION_SQL, UPSERT_AUTOROUTER_USER_SESSION_SQL]) async def test_a_sessionless_turn_writes_its_router_day_row_and_no_session_row(db, statement: str): - key = f"k-{uuid.uuid4()}" - router = f"auto-{uuid.uuid4()}" + key = f"k-{_new_identity()}" + router = f"auto-{_new_identity()}" for offset in range(2): await write_autorouter_turn( db, @@ -732,8 +775,8 @@ async def test_a_sessionless_turn_writes_its_router_day_row_and_no_session_row(d @pytest.mark.parametrize("historical", [True, False]) async def test_daily_token_coverage_stays_unknown_with_old_writers(db: Prisma, historical: bool) -> None: - key: Final = f"k-{uuid.uuid4()}" - router: Final = f"auto-{uuid.uuid4()}" + key: Final = f"k-{_new_identity()}" + router: Final = f"auto-{_new_identity()}" if historical: await db.execute_raw( 'INSERT INTO "LiteLLM_AutoRouterDailySpend" ' @@ -754,8 +797,8 @@ async def test_daily_token_coverage_stays_unknown_with_old_writers(db: Prisma, h @pytest.mark.parametrize("missing_first", [True, False]) async def test_missing_usage_never_completes_daily_token_coverage(db: Prisma, missing_first: bool) -> None: - key: Final = f"k-{uuid.uuid4()}" - router: Final = f"auto-{uuid.uuid4()}" + key: Final = f"k-{_new_identity()}" + router: Final = f"auto-{_new_identity()}" for offset, recorded in enumerate((not missing_first, missing_first)): await _turn( db, key, "A", T0 + timedelta(seconds=offset), router=router, @@ -769,8 +812,8 @@ async def test_missing_usage_never_completes_daily_token_coverage(db: Prisma, mi async def test_router_day_money_reconciles_with_the_overall_daily_total_including_sessionless_requests(db): from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, build_bulk_upsert, merge_by_conflict_key - key = f"k-{uuid.uuid4()}" - router = f"auto-{uuid.uuid4()}" + key = f"k-{_new_identity()}" + router = f"auto-{_new_identity()}" requests = (("session-1", 0.25, 1.5), ("session-1", 0.5, 2.0), ("", 0.1, 0.25)) for offset, (session_id, spend, saved) in enumerate(requests): await write_autorouter_turn( diff --git a/tests/proxy_behavior/spend/test_baseline_accounting.py b/tests/integration/spend/test_baseline_accounting.py similarity index 92% rename from tests/proxy_behavior/spend/test_baseline_accounting.py rename to tests/integration/spend/test_baseline_accounting.py index 4298b85cab4..60c5c32511a 100644 --- a/tests/proxy_behavior/spend/test_baseline_accounting.py +++ b/tests/integration/spend/test_baseline_accounting.py @@ -8,6 +8,7 @@ from datetime import datetime, timedelta, timezone from typing import Final import pytest +import pytest_asyncio from prisma import Prisma import litellm @@ -23,10 +24,17 @@ from litellm.proxy.db.create_views import SupportsRawQueries from litellm.proxy.spend_tracking.baseline_accounting import BaselineObservation from litellm.proxy.spend_tracking.savings import BaselineCostSnapshot from litellm.types.utils import Usage +from tests.integration._support.prisma_clients import connected_prisma pytestmark = pytest.mark.asyncio(loop_scope="session") +@pytest_asyncio.fixture(scope="module", loop_scope="session") +async def db() -> AsyncIterator[Prisma]: + async with connected_prisma() as client: + yield client + + @asynccontextmanager async def _transaction(db: Prisma, *, before_commit: bool = False, after_commit: bool = False) -> AsyncIterator[SupportsRawQueries]: async with db.tx() as tx: @@ -44,8 +52,8 @@ def _store(db: Prisma, **faults: bool) -> BaselineAccountingStore: return BaselineAccountingStore(transaction) -@pytest.fixture -def record() -> Callable[..., BaselineAccountingRecord]: +@pytest_asyncio.fixture(loop_scope="session") +async def record(db: Prisma) -> AsyncIterator[Callable[..., BaselineAccountingRecord]]: run: Final = uuid.uuid4().hex marker: Final = CountedBreakpoint("prefix", 3600, 6000, ("prefix",), "content", ("content",)) usage: Final = Usage( @@ -87,7 +95,21 @@ def record() -> Callable[..., BaselineAccountingRecord]: ), ) - return create + yield create + key: Final = run + scope: Final = "autorouter-baseline:v3:" + run * 2 + await db.execute_raw('DELETE FROM "LiteLLM_DailyTagSpend" WHERE api_key=$1', key) + await db.execute_raw('DELETE FROM "LiteLLM_DailyTeamSpend" WHERE api_key=$1', key) + await db.execute_raw('DELETE FROM "LiteLLM_DailyOrganizationSpend" WHERE api_key=$1', key) + await db.execute_raw('DELETE FROM "LiteLLM_DailyEndUserSpend" WHERE api_key=$1', key) + await db.execute_raw('DELETE FROM "LiteLLM_DailyAgentSpend" WHERE api_key=$1', key) + await db.execute_raw('DELETE FROM "LiteLLM_DailyUserSpend" WHERE api_key=$1', key) + await db.execute_raw('DELETE FROM "LiteLLM_AutoRouterDailySpend" WHERE api_key=$1', key) + await db.execute_raw('DELETE FROM "LiteLLM_AutoRouterUserSession" WHERE api_key=$1', key) + await db.execute_raw('DELETE FROM "LiteLLM_AutoRouterSession" WHERE api_key=$1', key) + await db.execute_raw('DELETE FROM "LiteLLM_AutoRouterBaselineObservation" WHERE scope=$1', scope) + await db.execute_raw('DELETE FROM "LiteLLM_AutoRouterBaselineComparison" WHERE scope=$1', scope) + await db.execute_raw('DELETE FROM "LiteLLM_SpendLogs" WHERE api_key=$1', key) async def _log(db: Prisma, record: BaselineAccountingRecord) -> None: diff --git a/tests/proxy_behavior/spend/test_cache_activity.py b/tests/integration/spend/test_cache_activity.py similarity index 93% rename from tests/proxy_behavior/spend/test_cache_activity.py rename to tests/integration/spend/test_cache_activity.py index 528764fbf45..c381d51c402 100644 --- a/tests/proxy_behavior/spend/test_cache_activity.py +++ b/tests/integration/spend/test_cache_activity.py @@ -7,10 +7,13 @@ tests/unit/proxy/analytics_endpoints/test_analytics_endpoints.py. import json import uuid +from collections.abc import AsyncIterator from datetime import datetime from typing import Final import pytest +import pytest_asyncio +from prisma import Prisma from litellm.proxy.analytics_endpoints.cache_activity import ( ERROR_BREAKDOWN_SQL, @@ -19,6 +22,7 @@ from litellm.proxy.analytics_endpoints.cache_activity import ( KEY_ALIAS_OPTIONS_SQL, MODEL_OPTIONS_SQL, ) +from tests.integration._support.prisma_clients import connected_prisma pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -34,6 +38,12 @@ INFO_ONLY_MODEL: Final = f"ghost-model-{RUN}" NO_FILTER: Final = "[]" +@pytest_asyncio.fixture(scope="module", loop_scope="session") +async def db() -> AsyncIterator[Prisma]: + async with connected_prisma() as client: + yield client + + async def _spend_log(db, api_key: str, call_type: str, status: str, model: str = "", error_code: str = "") -> None: metadata: Final = {"error_information": {"error_code": error_code, "error_class": "ProxyException"}} await db.execute_raw( diff --git a/tests/load_tests/conftest.py b/tests/load_tests/conftest.py deleted file mode 100644 index 48a98663e4e..00000000000 --- a/tests/load_tests/conftest.py +++ /dev/null @@ -1,5 +0,0 @@ -from tests.load_tests.memory_leak_utils import ( # noqa: F401 # re-exported so pytest resolves these fixtures by name - limit_memory, - mock_server, - test_router, -) diff --git a/tests/load_tests/memory_leak_utils.py b/tests/load_tests/memory_leak_utils.py deleted file mode 100644 index fae294e5a2c..00000000000 --- a/tests/load_tests/memory_leak_utils.py +++ /dev/null @@ -1,328 +0,0 @@ -""" -Memory Leak Testing Utilities - -This module provides reusable utilities, fixtures, and helpers for memory leak -and OOM (Out of Memory) detection tests. It includes: -- Mock server setup for local testing -- Memory tracking fixtures -- Router fixtures configured for testing -- Helper functions for running memory baseline tests - -Usage: - from tests.load_tests.memory_leak_utils import ( - mock_server, - limit_memory, - test_router, - run_memory_baseline_test, - ) -""" - -import gc -import os -import socket -import sys -import time -from threading import Thread - -# Add parent directory to path to import litellm (same pattern as other tests) -filepath = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, os.path.abspath(os.path.join(filepath, "../.."))) - -import httpx -import pytest -import psutil -from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse -from litellm.router import Router - -# Test Configuration Constants -TEST_API_KEY = "sk-9876" -TEST_MODEL_NAME = "gpt-3.5-turbo" - -# Timing Constants (seconds) -GC_STABILIZATION_DELAY = 0.05 - - -# Mock OpenAI-compatible server -def create_mock_server(): - """Create a simple FastAPI mock server that mimics OpenAI API responses.""" - app = FastAPI() - - @app.post("/v1/chat/completions") - @app.post("/chat/completions") - async def chat_completions(request: Request): - """Mock OpenAI chat completions endpoint.""" - request_data = await request.json() - # Return a simple mock response - return JSONResponse( - { - "id": "chatcmpl-mock", - "object": "chat.completion", - "created": int(time.time()), - "model": request_data.get("model", TEST_MODEL_NAME), - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "Mock response"}, - "finish_reason": "stop", - } - ], - "usage": { - "prompt_tokens": 10, - "completion_tokens": 5, - "total_tokens": 15, - }, - } - ) - - # Catch-all route to see what URLs are being requested - @app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"]) - async def catch_all(request: Request, path: str): - """Catch-all route to debug what URLs are being requested.""" - print(f"[Mock Server] Received request: {request.method} {request.url.path}") - # For non-chat-completions, return 404 - return JSONResponse({"detail": "Not Found"}, status_code=404) - - return app - - -def run_server(app, port): - """Run uvicorn server in a thread.""" - import uvicorn - - # Use uvicorn.run which blocks - this is fine in a daemon thread - uvicorn.run(app, host="127.0.0.1", port=port, log_level="error", access_log=False) - - -@pytest.fixture(scope="session") -def mock_server(): - """Start a mock server in a separate thread for the test session. - - Yields the server URL (with trailing slash) for use in router configuration. - """ - app = create_mock_server() - port = 18888 - - # Check if port is already in use - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - sock.bind(("127.0.0.1", port)) - sock.close() - except OSError: - # Port already in use, try next port - port = 18889 - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - sock.bind(("127.0.0.1", port)) - sock.close() - except OSError: - pytest.fail( - f"Could not find available port for mock server (tried 18888, 18889)" - ) - - # Start server in background thread - thread = Thread(target=lambda: run_server(app, port), daemon=True) - thread.start() - - # Wait for server to start and verify it's accessible - # Ensure api_base has trailing slash (LiteLLM appends /v1/chat/completions) - server_url = f"http://127.0.0.1:{port}/" - max_attempts = 20 # More attempts to ensure server is ready - server_ready = False - for attempt in range(max_attempts): - time.sleep(0.3) # Longer wait between attempts - try: - # Test the actual endpoint we'll use (LiteLLM appends /v1/chat/completions to api_base) - response = httpx.post( - f"{server_url}v1/chat/completions", - json={ - "model": TEST_MODEL_NAME, - "messages": [{"role": "user", "content": "test"}], - }, - timeout=2.0, - ) - if response.status_code == 200: - server_ready = True - print(f"[Mock Server] Server ready at {server_url}") - break - except (httpx.ConnectError, httpx.TimeoutException, httpx.NetworkError) as e: - # Server not ready yet, continue waiting - if attempt == max_attempts - 1: - pytest.fail( - f"Mock server failed to start on {server_url} after {max_attempts} attempts. " - f"Could not connect to /v1/chat/completions endpoint. Error: {e}" - ) - continue - except Exception as e: - # Other errors might indicate server is up but endpoint has issues - # If we get a response (even error), server is running - print(f"[Mock Server] Server responded with error (but is running): {e}") - server_ready = True - break - - if not server_ready: - pytest.fail( - f"Mock server not accessible at {server_url} after {max_attempts} attempts" - ) - - yield server_url - - # Server will be cleaned up when thread dies (daemon=True) - - -@pytest.fixture -def limit_memory(request): - """Fixture to track memory usage and enforce limits via @pytest.mark.limit_leaks marker. - - Usage: - @pytest.mark.limit_leaks("40 MB") - def test_something(limit_memory): - # Test code here - # Memory will be measured and test will fail if increase exceeds limit - """ - marker = request.node.get_closest_marker("limit_leaks") - if marker: - # Parse limit from marker (e.g., "30 MB" -> 30) - limit_str = marker.args[0] if marker.args else "100 MB" - limit_mb = float(limit_str.split()[0]) - limit_bytes = limit_mb * 1024 * 1024 - - # Measure baseline memory (router will be fresh from fixture) - process = psutil.Process(os.getpid()) - baseline_memory = process.memory_info().rss - - yield - - # Force GC before measuring final memory - gc.collect() - # Small delay for memory to stabilize - time.sleep(GC_STABILIZATION_DELAY) - - # Measure final memory after test - final_memory = process.memory_info().rss - memory_increase = final_memory - baseline_memory - memory_increase_mb = memory_increase / 1024 / 1024 - - # Print memory stats - print(f"\n[Memory Limit Test] Memory usage:") - print(f" Baseline: {baseline_memory / 1024 / 1024:.2f} MB") - print(f" Final: {final_memory / 1024 / 1024:.2f} MB") - print(f" Increase: {memory_increase_mb:+.2f} MB") - print(f" Limit: {limit_mb:.2f} MB") - - # Fail if memory increase exceeds limit - if memory_increase > limit_bytes: - pytest.fail( - f"Memory limit exceeded: {memory_increase_mb:.2f} MB increase > {limit_mb:.2f} MB limit. " - f"Baseline: {baseline_memory / 1024 / 1024:.2f} MB, Final: {final_memory / 1024 / 1024:.2f} MB" - ) - else: - yield - - -@pytest.fixture -def test_router(mock_server): - """Fixture to create a fresh router instance for each test. - - Uses the mock server fixture to avoid external API calls. - Disables cooldowns to prevent deployments from being marked unavailable. - - Usage: - def test_something(test_router, limit_memory): - # Use test_router for making requests - response = await test_router.acompletion(...) - """ - router = Router( - model_list=[ - { - "model_name": TEST_MODEL_NAME, - "litellm_params": { - "model": f"openai/{TEST_MODEL_NAME}", - "api_base": mock_server, - "api_key": TEST_API_KEY, - }, - }, - ], - disable_cooldowns=True, # Disable cooldowns for testing - allowed_fails=1000, # Allow many failures before cooldown (effectively disabled) - ) - yield router - # Cleanup after test - try: - router.discard() - except Exception: - pass # Ignore cleanup errors - - -async def run_memory_baseline_test(num_requests: int, router: Router, limit_memory): - """Helper function to run memory baseline test with specified number of requests. - - Makes requests concurrently in batches for speed, with proper error handling - that doesn't fail the test on individual request failures. - - Args: - num_requests: Number of requests to make. - router: Router instance to use for requests. - limit_memory: Pytest fixture for memory tracking (reference to suppress linter warning). - - Example: - @pytest.mark.asyncio - @pytest.mark.limit_leaks("40 MB") - async def test_memory(test_router, limit_memory): - await run_memory_baseline_test(1000, test_router, limit_memory) - """ - # Fixture is used automatically by pytest - reference it to suppress linter warning - _ = limit_memory - - # Make requests concurrently in batches for speed - # Batch size of 20 provides good balance between speed and memory pressure - BATCH_SIZE = 20 - - for batch_start in range(0, num_requests, BATCH_SIZE): - batch_end = min(batch_start + BATCH_SIZE, num_requests) - # Create concurrent tasks for this batch - tasks = [ - router.acompletion( - model=TEST_MODEL_NAME, - messages=[{"role": "user", "content": f"Test request {i}"}], - ) - for i in range(batch_start, batch_end) - ] - # Execute batch concurrently - # Note: return_exceptions=True allows test to continue even if some requests fail - import asyncio - - responses = await asyncio.gather(*tasks, return_exceptions=True) - # Filter out failed requests but continue with test - valid_responses = [] - failed_count = 0 - for i, response in enumerate(responses): - if isinstance(response, Exception): - failed_count += 1 - # Log exception but continue - print( - f" Warning: Request {batch_start + i} failed: {type(response).__name__}: {response}" - ) - elif response is None: - failed_count += 1 - print(f" Warning: Request {batch_start + i} returned None") - else: - valid_responses.append(response) - - # Continue with valid responses - don't fail the test - # If all failed, that's logged but test continues (might indicate bigger issue) - if failed_count > 0: - print( - f" Note: {failed_count}/{len(responses)} requests failed in batch {batch_start}-{batch_end}, continuing with {len(valid_responses)} valid responses" - ) - - # Use valid_responses for cleanup - responses = valid_responses - # Clean up batch - del responses - del tasks - del valid_responses - # GC after each batch to prevent accumulation - gc.collect() - - print(f"[Simple Memory Test] Completed {num_requests} requests") diff --git a/tests/load_tests/test_datadog_load_test.py b/tests/load_tests/test_datadog_load_test.py deleted file mode 100644 index 3dfc3fc6da4..00000000000 --- a/tests/load_tests/test_datadog_load_test.py +++ /dev/null @@ -1,102 +0,0 @@ -import os - - -import asyncio -import litellm -import pytest -import logging -from litellm._logging import verbose_logger - - -def test_datadog_logging_async(): - try: - # litellm.set_verbose = True - os.environ["DD_API_KEY"] = "anything" - os.environ["_DATADOG_BASE_URL"] = ( - "https://exampleopenaiendpoint-production.up.railway.app" - ) - - os.environ["DD_SITE"] = "us5.datadoghq.com" - os.environ["DD_API_KEY"] = "xxxxxx" - - litellm.success_callback = ["datadog"] - - percentage_diffs = [] - - for run in range(1): - print(f"\nRun {run + 1}:") - - # Test with empty success_callback - litellm.success_callback = [] - litellm.callbacks = [] - start_time_empty_callback = asyncio.run(make_async_calls()) - print("Done with no callback test") - - # Test with datadog callback - print("Starting datadog test") - litellm.success_callback = ["datadog"] - start_time_datadog = asyncio.run(make_async_calls()) - print("Done with datadog test") - - # Compare times and calculate percentage difference - print(f"Time with success_callback='datadog': {start_time_datadog}") - print(f"Time with empty success_callback: {start_time_empty_callback}") - - percentage_diff = ( - abs(start_time_datadog - start_time_empty_callback) - / start_time_empty_callback - * 100 - ) - percentage_diffs.append(percentage_diff) - print(f"Performance difference: {percentage_diff:.2f}%") - - print("percentage_diffs", percentage_diffs) - avg_percentage_diff = sum(percentage_diffs) / len(percentage_diffs) - print(f"\nAverage performance difference: {avg_percentage_diff:.2f}%") - - assert ( - avg_percentage_diff < 10 - ), f"Average performance difference of {avg_percentage_diff:.2f}% exceeds 10% threshold" - - except litellm.Timeout: - pass - except Exception as e: - pytest.fail(f"An exception occurred - {e}") - - -async def make_async_calls(metadata=None, **completion_kwargs): - total_tasks = 300 - batch_size = 100 - total_time = 0 - - for batch in range(1): - tasks = [create_async_task() for _ in range(batch_size)] - - start_time = asyncio.get_event_loop().time() - responses = await asyncio.gather(*tasks) - - for idx, response in enumerate(responses): - print(f"Response from Task {batch * batch_size + idx + 1}: {response}") - - await asyncio.sleep(7) - - batch_time = asyncio.get_event_loop().time() - start_time - total_time += batch_time - - return total_time - - -def create_async_task(**completion_kwargs): - litellm.set_verbose = True - completion_args = { - "model": "openai/chatgpt-v-3", - "api_version": "2024-02-01", - "messages": [{"role": "user", "content": "This is a test"}], - "max_tokens": 5, - "temperature": 0.7, - "timeout": 5, - "user": "datadog_latency_test_user", - "mock_response": "hello from my load test", - } - completion_args.update(completion_kwargs) - return asyncio.create_task(litellm.acompletion(**completion_args)) diff --git a/tests/load_tests/test_langsmith_load_test.py b/tests/load_tests/test_langsmith_load_test.py deleted file mode 100644 index 5eca0e339f5..00000000000 --- a/tests/load_tests/test_langsmith_load_test.py +++ /dev/null @@ -1,109 +0,0 @@ - -import os - - -import asyncio -import litellm -from litellm._logging import verbose_logger -import logging -import time -import pytest - - -def test_langsmith_logging_async(): - try: - os.environ["LANGSMITH_API_KEY"] = "lsv2_anything" - os.environ["LANGSMITH_PROJECT"] = "pr-b" - os.environ["LANGSMITH_BASE_URL"] = ( - "https://exampleopenaiendpoint-production.up.railway.app" - ) - - percentage_diffs = [] - - for run in range(3): - print(f"\nRun {run + 1}:") - - # Test with empty success_callback - litellm.success_callback = [] - litellm.callbacks = [] - litellm._async_success_callback = [] - litellm._async_failure_callback = [] - litellm.failure_callback = [] - start_time_empty_callback = asyncio.run(make_async_calls()) - print("Done with no callback test") - - # Test with langsmith callback - print("Starting langsmith test") - litellm.success_callback = ["langsmith"] - start_time_langsmith = asyncio.run(make_async_calls()) - print("Done with langsmith test") - - # Compare times and calculate percentage difference - print(f"Time with success_callback='langsmith': {start_time_langsmith}") - print(f"Time with empty success_callback: {start_time_empty_callback}") - - percentage_diff = ( - abs(start_time_langsmith - start_time_empty_callback) - / start_time_empty_callback - * 100 - ) - percentage_diffs.append(percentage_diff) - print(f"Performance difference: {percentage_diff:.2f}%") - print("percentage_diffs", percentage_diffs) - # Calculate average percentage difference - avg_percentage_diff = sum(percentage_diffs) / len(percentage_diffs) - print(f"\nAverage performance difference: {avg_percentage_diff:.2f}%") - - # Assert that the average difference is not more than 10% - assert ( - avg_percentage_diff < 10 - ), f"Average performance difference of {avg_percentage_diff:.2f}% exceeds 10% threshold" - - except litellm.Timeout as e: - pass - except Exception as e: - pytest.fail(f"An exception occurred - {e}") - - -async def make_async_calls(metadata=None, **completion_kwargs): - total_tasks = 300 - batch_size = 100 - total_time = 0 - - for batch in range(3): - tasks = [create_async_task() for _ in range(batch_size)] - - start_time = asyncio.get_event_loop().time() - responses = await asyncio.gather(*tasks) - - for idx, response in enumerate(responses): - print(f"Response from Task {batch * batch_size + idx + 1}: {response}") - - await asyncio.sleep(1) - - batch_time = asyncio.get_event_loop().time() - start_time - total_time += batch_time - - return total_time - - -def create_async_task(**completion_kwargs): - """ - Creates an async task for the litellm.acompletion function. - This is just the task, but it is not run here. - To run the task it must be awaited or used in other asyncio coroutine execution functions like asyncio.gather. - Any kwargs passed to this function will be passed to the litellm.acompletion function. - By default a standard set of arguments are used for the litellm.acompletion function. - """ - completion_args = { - "model": "openai/chatgpt-v-2", - "api_version": "2024-02-01", - "messages": [{"role": "user", "content": "This is a test"}], - "max_tokens": 5, - "temperature": 0.7, - "timeout": 5, - "user": "langfuse_latency_test_user", - "mock_response": "hello from my load test", - } - completion_args.update(completion_kwargs) - return asyncio.create_task(litellm.acompletion(**completion_args)) diff --git a/tests/load_tests/test_linear_memory_growth.py b/tests/load_tests/test_linear_memory_growth.py deleted file mode 100644 index f1c36924a2a..00000000000 --- a/tests/load_tests/test_linear_memory_growth.py +++ /dev/null @@ -1,115 +0,0 @@ -""" -Memory Leak Detection Tests - Linear Memory Growth - -Tests that check for linear/progressive memory growth by running different numbers -of requests (1k, 2k, 4k, 10k, 30k) with the same memory limit. If lower request -count tests pass but higher ones fail, it indicates linear memory growth per request. - -These tests will fail if memory leaks are detected, helping catch OOM issues before production. - -IMPORTANT: These tests should be run INDIVIDUALLY, not all together. Running them -together causes memory baseline drift between tests, making it difficult to detect -linear growth accurately. Each test should be run in isolation: - - pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_1k -v - pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_2k -v - # etc. - -NOTE: Not recommended for accurate results: -pytest tests/load_tests/test_linear_memory_growth.py -v -""" - -import pytest - -from tests.load_tests.memory_leak_utils import run_memory_baseline_test - -# Memory limit for all linear memory growth tests -MEMORY_LIMIT = "40 MB" - - -@pytest.mark.asyncio -@pytest.mark.limit_leaks(MEMORY_LIMIT) -@pytest.mark.no_parallel # Must run sequentially - measures process memory -async def test_memory_baseline_1k(test_router, limit_memory): - """ - Memory baseline test with 1,000 requests. - Uses @pytest.mark.limit_leaks("40 MB") to enforce memory limit. - If this passes but higher request count tests fail, indicates progressive memory leak. - - NOTE: This test should be run INDIVIDUALLY, not with other tests in this file. - Running multiple tests together causes memory baseline drift, making it difficult - to accurately detect linear memory growth. Run with: - pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_1k -v - """ - await run_memory_baseline_test(1000, test_router, limit_memory) - - -@pytest.mark.asyncio -@pytest.mark.limit_leaks(MEMORY_LIMIT) -@pytest.mark.no_parallel # Must run sequentially - measures process memory -async def test_memory_baseline_2k(test_router, limit_memory): - """ - Memory baseline test with 2,000 requests. - Uses @pytest.mark.limit_leaks("40 MB") to enforce memory limit. - If this passes but test_memory_baseline_4k fails, indicates progressive memory leak. - - NOTE: This test should be run INDIVIDUALLY, not with other tests in this file. - Running multiple tests together causes memory baseline drift, making it difficult - to accurately detect linear memory growth. Run with: - pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_2k -v - """ - await run_memory_baseline_test(2000, test_router, limit_memory) - - -@pytest.mark.asyncio -@pytest.mark.limit_leaks(MEMORY_LIMIT) -@pytest.mark.no_parallel # Must run sequentially - measures process memory -async def test_memory_baseline_4k(test_router, limit_memory): - """ - Memory baseline test with 4,000 requests. - Uses @pytest.mark.limit_leaks("40 MB") to enforce memory limit. - If test_memory_baseline_1k and test_memory_baseline_2k pass but this fails, - it's a clear sign of sequential/progressive memory growth. - - NOTE: This test should be run INDIVIDUALLY, not with other tests in this file. - Running multiple tests together causes memory baseline drift, making it difficult - to accurately detect linear memory growth. Run with: - pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_4k -v - """ - await run_memory_baseline_test(4000, test_router, limit_memory) - - -@pytest.mark.asyncio -@pytest.mark.limit_leaks(MEMORY_LIMIT) -@pytest.mark.no_parallel # Must run sequentially - measures process memory -async def test_memory_baseline_10k(test_router, limit_memory): - """ - Memory baseline test with 10,000 requests. - Uses @pytest.mark.limit_leaks("40 MB") to enforce memory limit. - If test_memory_baseline_1k and test_memory_baseline_2k pass but this fails, - it's a clear sign of sequential/progressive memory growth. - - NOTE: This test should be run INDIVIDUALLY, not with other tests in this file. - Running multiple tests together causes memory baseline drift, making it difficult - to accurately detect linear memory growth. Run with: - pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_10k -v - """ - await run_memory_baseline_test(10000, test_router, limit_memory) - - -@pytest.mark.asyncio -@pytest.mark.limit_leaks(MEMORY_LIMIT) -@pytest.mark.no_parallel # Must run sequentially - measures process memory -async def test_memory_baseline_30k(test_router, limit_memory): - """ - Memory baseline test with 30,000 requests. - Uses @pytest.mark.limit_leaks("40 MB") to enforce memory limit. - If test_memory_baseline_1k and test_memory_baseline_2k pass but this fails, - it's a clear sign of sequential/progressive memory growth. - - NOTE: This test should be run INDIVIDUALLY, not with other tests in this file. - Running multiple tests together causes memory baseline drift, making it difficult - to accurately detect linear memory growth. Run with: - pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_30k -v - """ - await run_memory_baseline_test(30000, test_router, limit_memory) diff --git a/tests/load_tests/test_memory_usage.py b/tests/load_tests/test_memory_usage.py deleted file mode 100644 index c5b5134a3d7..00000000000 --- a/tests/load_tests/test_memory_usage.py +++ /dev/null @@ -1,237 +0,0 @@ -import asyncio -import os -import sys -import traceback -import tracemalloc - -from dotenv import load_dotenv - -load_dotenv() -import io - - - -import litellm.types -import litellm.types.utils -from litellm.router import Router -from typing import Optional -from unittest.mock import MagicMock, patch - -import pytest -import litellm -from typing import Callable, Any - -import gc -from typing import Type -from pydantic import BaseModel - -from litellm.proxy.proxy_server import app - - -async def get_memory_usage() -> float: - """Get current memory usage of the process in MB""" - import psutil - - process = psutil.Process(os.getpid()) - return process.memory_info().rss / 1024 / 1024 - - -async def run_memory_test(request_func: Callable, name: str) -> None: - """ - Generic memory test function - Args: - request_func: Async function that makes the API request - name: Name of the test for logging - """ - memory_before = await get_memory_usage() - print(f"\n{name} - Initial memory usage: {memory_before:.2f}MB") - - for i in range(60 * 4): # 4 minutes - all_tasks = [request_func() for _ in range(100)] - await asyncio.gather(*all_tasks) - current_memory = await get_memory_usage() - print(f"Request {i * 100}: Current memory usage: {current_memory:.2f}MB") - - memory_after = await get_memory_usage() - print(f"Final memory usage: {memory_after:.2f}MB") - - memory_diff = memory_after - memory_before - print(f"Memory difference: {memory_diff:.2f}MB") - - assert memory_diff < 10, f"Memory increased by {memory_diff:.2f}MB" - - -async def make_completion_request(): - return await litellm.acompletion( - model="openai/gpt-4o", - messages=[{"role": "user", "content": "Test message for memory usage"}], - api_base="https://exampleopenaiendpoint-production.up.railway.app/", - ) - - -async def make_text_completion_request(): - return await litellm.atext_completion( - model="openai/gpt-4o", - prompt="Test message for memory usage", - api_base="https://exampleopenaiendpoint-production.up.railway.app/", - ) - - -@pytest.mark.asyncio -@pytest.mark.skip( - reason="This test is too slow to run on every commit. We can use this after nightly release" -) -async def test_acompletion_memory(): - """Test memory usage for litellm.acompletion""" - await run_memory_test(make_completion_request, "acompletion") - - -@pytest.mark.asyncio -@pytest.mark.skip( - reason="This test is too slow to run on every commit. We can use this after nightly release" -) -async def test_atext_completion_memory(): - """Test memory usage for litellm.atext_completion""" - await run_memory_test(make_text_completion_request, "atext_completion") - - -litellm_router = Router( - model_list=[ - { - "model_name": "text-gpt-4o", - "litellm_params": { - "model": "text-completion-openai/gpt-3.5-turbo-instruct-unlimited", - "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", - }, - }, - { - "model_name": "chat-gpt-4o", - "litellm_params": { - "model": "openai/gpt-4o", - "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", - }, - }, - ] -) - - -async def make_router_atext_completion_request(): - return await litellm_router.atext_completion( - model="text-gpt-4o", - temperature=0.5, - frequency_penalty=0.5, - prompt="<|fim prefix|> Test message for memory usage <|fim prefix|> Test message for memory usage", - api_base="https://exampleopenaiendpoint-production.up.railway.app/", - max_tokens=500, - ) - - -@pytest.mark.asyncio -@pytest.mark.skip( - reason="This test is too slow to run on every commit. We can use this after nightly release" -) -async def test_router_atext_completion_memory(): - """Test memory usage for litellm.atext_completion""" - await run_memory_test( - make_router_atext_completion_request, "router_atext_completion" - ) - - -async def make_router_acompletion_request(): - return await litellm_router.acompletion( - model="chat-gpt-4o", - messages=[{"role": "user", "content": "Test message for memory usage"}], - api_base="https://exampleopenaiendpoint-production.up.railway.app/", - ) - - -def get_pydantic_objects(): - """Get all Pydantic model instances in memory""" - return [obj for obj in gc.get_objects() if isinstance(obj, BaseModel)] - - -def analyze_pydantic_snapshot(): - """Analyze current Pydantic objects""" - objects = get_pydantic_objects() - type_counts = {} - - for obj in objects: - type_name = type(obj).__name__ - type_counts[type_name] = type_counts.get(type_name, 0) + 1 - - print("\nPydantic Object Count:") - for type_name, count in sorted( - type_counts.items(), key=lambda x: x[1], reverse=True - ): - print(f"{type_name}: {count}") - # Print an example object if helpful - if count > 1000: # Only look at types with many instances - example = next(obj for obj in objects if type(obj).__name__ == type_name) - print(f"Example fields: {example.dict().keys()}") - - -from collections import defaultdict - - -def get_blueprint_stats(): - # Dictionary to collect lists of blueprint objects by their type name. - blueprint_objects = defaultdict(list) - - for obj in gc.get_objects(): - try: - # Check for attributes that are typically present on Pydantic model blueprints. - if ( - hasattr(obj, "__pydantic_fields__") - or hasattr(obj, "__pydantic_validator__") - or hasattr(obj, "__pydantic_core_schema__") - ): - typename = type(obj).__name__ - blueprint_objects[typename].append(obj) - except Exception: - # Some objects might cause issues when inspected; skip them. - continue - - # Now calculate count and total shallow size for each type. - stats = [] - for typename, objs in blueprint_objects.items(): - total_size = sum(sys.getsizeof(o) for o in objs) - stats.append((typename, len(objs), total_size)) - return stats - - -def print_top_blueprints(top_n=10): - stats = get_blueprint_stats() - # Sort by total_size in descending order. - stats.sort(key=lambda x: x[2], reverse=True) - - print(f"Top {top_n} Pydantic blueprint objects by memory usage (shallow size):") - for typename, count, total_size in stats[:top_n]: - print( - f"{typename}: count = {count}, total shallow size = {total_size / 1024:.2f} KiB" - ) - - # Get one instance of the blueprint object for this type (if available) - blueprint_objs = [ - obj for obj in gc.get_objects() if type(obj).__name__ == typename - ] - if blueprint_objs: - obj = blueprint_objs[0] - # Ensure that tracemalloc is enabled and tracking this allocation. - tb = tracemalloc.get_object_traceback(obj) - if tb: - print("Allocation traceback (most recent call last):") - for frame in tb.format(): - print(frame) - else: - print("No allocation traceback available for this object.") - else: - print("No blueprint objects found for this type.") - - -@pytest.fixture(autouse=True) -def cleanup(): - """Cleanup after each test""" - import gc - - yield - gc.collect() diff --git a/tests/load_tests/test_otel_load_test.py b/tests/load_tests/test_otel_load_test.py deleted file mode 100644 index 57dcc53a50b..00000000000 --- a/tests/load_tests/test_otel_load_test.py +++ /dev/null @@ -1,97 +0,0 @@ - -import os - - -import asyncio -import litellm -from litellm._logging import verbose_logger -import logging -import time -import pytest - - -def test_otel_logging_async(): - try: - os.environ["OTEL_EXPORTER"] = "otlp_http" - os.environ["OTEL_ENDPOINT"] = ( - "https://exampleopenaiendpoint-production.up.railway.app/traces" - ) - os.environ["OTEL_HEADERS"] = "Authorization=K0BSwd" - - def single_run(): - litellm.callbacks = [] - start_time_empty = asyncio.run(make_async_calls()) - print(f"Time with empty callback: {start_time_empty}") - - litellm.callbacks = ["otel"] - start_time_otel = asyncio.run(make_async_calls()) - print(f"Time with otel callback: {start_time_otel}") - - percent_diff = ( - abs(start_time_otel - start_time_empty) / start_time_empty * 100 - ) - print(f"Run performance difference: {percent_diff:.2f}%") - return percent_diff - - percent_diffs = [single_run() for _ in range(3)] - avg_percent_diff = sum(percent_diffs) / len(percent_diffs) - - print(f"Percentage differences: {percent_diffs}") - print(f"Average performance difference: {avg_percent_diff:.2f}%") - - assert ( - avg_percent_diff < 30 - ), f"Average performance difference of {avg_percent_diff:.2f}% exceeds 30% threshold" - - except litellm.Timeout as e: - pass - except Exception as e: - pytest.fail(f"An exception occurred - {e}") - - -async def make_async_calls(metadata=None, **completion_kwargs): - total_start_time = asyncio.get_event_loop().time() - tasks = [] - - async def create_and_run_task(): - task = create_async_task(**completion_kwargs) - response = await task - print(f"Response: {response}") - - for _ in range(3): # Run for 10 seconds - # Create 100 tasks - tasks = [] - for _ in range(100): - tasks.append(asyncio.create_task(create_and_run_task())) - - # Wait for any remaining tasks to complete - await asyncio.gather(*tasks) - - await asyncio.sleep(1) - - # Calculate the total time taken - total_time = asyncio.get_event_loop().time() - total_start_time - - return total_time - - -def create_async_task(**completion_kwargs): - """ - Creates an async task for the litellm.acompletion function. - This is just the task, but it is not run here. - To run the task it must be awaited or used in other asyncio coroutine execution functions like asyncio.gather. - Any kwargs passed to this function will be passed to the litellm.acompletion function. - By default a standard set of arguments are used for the litellm.acompletion function. - """ - completion_args = { - "model": "openai/chatgpt-v-3", - "api_version": "2024-02-01", - "messages": [{"role": "user", "content": "This is a test" * 100}], - "max_tokens": 5, - "temperature": 0.7, - "timeout": 5, - "user": "langfuse_latency_test_user", - "mock_response": "Mock response", - } - completion_args.update(completion_kwargs) - return asyncio.create_task(litellm.acompletion(**completion_args)) diff --git a/tests/load_tests/test_vertex_embeddings_load_test.py b/tests/load_tests/test_vertex_embeddings_load_test.py deleted file mode 100644 index c5b9a80ec6b..00000000000 --- a/tests/load_tests/test_vertex_embeddings_load_test.py +++ /dev/null @@ -1,120 +0,0 @@ -""" -Load test on vertex AI embeddings to ensure vertex median response time is less than 300ms - -""" - -import os - - -import asyncio -import litellm -import pytest -import time -from statistics import mean, median -import json -import tempfile - - -def load_vertex_ai_credentials(): - # Define the path to the vertex_key.json file - print("loading vertex ai credentials") - filepath = os.path.dirname(os.path.abspath(__file__)) - vertex_key_path = filepath + "/vertex_key.json" - - # Read the existing content of the file or create an empty dictionary - try: - with open(vertex_key_path, "r") as file: - # Read the file content - print("Read vertexai file path") - content = file.read() - - # If the file is empty or not valid JSON, create an empty dictionary - if not content or not content.strip(): - service_account_key_data = {} - else: - # Attempt to load the existing JSON content - file.seek(0) - service_account_key_data = json.load(file) - except FileNotFoundError: - # If the file doesn't exist, create an empty dictionary - service_account_key_data = {} - - # Update the service_account_key_data with environment variables - private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "") - private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "") - private_key = private_key.replace("\\n", "\n") - service_account_key_data["private_key_id"] = private_key_id - service_account_key_data["private_key"] = private_key - - # Create a temporary file - with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file: - # Write the updated content to the temporary files - json.dump(service_account_key_data, temp_file, indent=2) - - # Export the temporary file as GOOGLE_APPLICATION_CREDENTIALS - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name) - - -async def create_async_vertex_embedding_task(): - load_vertex_ai_credentials() - base_url = "https://exampleopenaiendpoint-production.up.railway.app/v1/projects/pathrise-convert-1606954137718/locations/us-central1/publishers/google/models/textembedding-gecko@001" - embedding_args = { - "model": "vertex_ai/textembedding-gecko", - "input": "This is a test sentence for embedding.", - "timeout": 10, - "api_base": base_url, - } - start_time = time.time() - response = await litellm.aembedding(**embedding_args) - end_time = time.time() - print(f"Vertex AI embedding time: {end_time - start_time:.2f} seconds") - return response, end_time - start_time - - -async def run_load_test(duration_seconds, requests_per_second): - end_time = time.time() + duration_seconds - vertex_times = [] - - print( - f"Running Load Test for {duration_seconds} seconds at {requests_per_second} RPS..." - ) - while time.time() < end_time: - vertex_tasks = [ - create_async_vertex_embedding_task() for _ in range(requests_per_second) - ] - - vertex_results = await asyncio.gather(*vertex_tasks) - - vertex_times.extend([duration for _, duration in vertex_results]) - - # Sleep for 1 second to maintain the desired RPS - await asyncio.sleep(1) - - return vertex_times - - -def analyze_results(vertex_times): - median_vertex = median(vertex_times) - print(f"Vertex AI median response time: {median_vertex:.4f} seconds") - - if median_vertex > 3: - pytest.fail( - f"Vertex AI median response time is greater than 500ms: {median_vertex:.4f} seconds" - ) - else: - print("Performance is good") - return True - - -@pytest.mark.asyncio -async def test_embedding_performance(monkeypatch): - """ - Run load test on vertex AI embeddings to ensure vertex median response time is less than 300ms - - 20 RPS for 20 seconds - """ - monkeypatch.setattr(litellm, "api_base", None) - duration_seconds = 20 - requests_per_second = 20 - vertex_times = await run_load_test(duration_seconds, requests_per_second) - result = analyze_results(vertex_times) diff --git a/tests/load_tests/test_vertex_load_tests.py b/tests/load_tests/test_vertex_load_tests.py deleted file mode 100644 index 93e1ed24f72..00000000000 --- a/tests/load_tests/test_vertex_load_tests.py +++ /dev/null @@ -1,147 +0,0 @@ -import os - - -import asyncio -import litellm -import pytest -import time -import json -import tempfile -from dotenv import load_dotenv - - -def load_vertex_ai_credentials(): - # Define the path to the vertex_key.json file - print("loading vertex ai credentials") - filepath = os.path.dirname(os.path.abspath(__file__)) - vertex_key_path = filepath + "/vertex_key.json" - - # Read the existing content of the file or create an empty dictionary - try: - with open(vertex_key_path, "r") as file: - # Read the file content - print("Read vertexai file path") - content = file.read() - - # If the file is empty or not valid JSON, create an empty dictionary - if not content or not content.strip(): - service_account_key_data = {} - else: - # Attempt to load the existing JSON content - file.seek(0) - service_account_key_data = json.load(file) - except FileNotFoundError: - # If the file doesn't exist, create an empty dictionary - service_account_key_data = {} - - # Update the service_account_key_data with environment variables - private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "") - private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "") - private_key = private_key.replace("\\n", "\n") - service_account_key_data["private_key_id"] = private_key_id - service_account_key_data["private_key"] = private_key - - # Create a temporary file - with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file: - # Write the updated content to the temporary files - json.dump(service_account_key_data, temp_file, indent=2) - - # Export the temporary file as GOOGLE_APPLICATION_CREDENTIALS - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name) - - -@pytest.mark.asyncio -async def test_vertex_load(): - try: - load_vertex_ai_credentials() - percentage_diffs = [] - - for run in range(3): - print(f"\nRun {run + 1}:") - - # Test with text-only message - start_time_text = await make_async_calls(message_type="text") - print("Done with text-only message test") - - # Test with text + image message - start_time_image = await make_async_calls(message_type="image") - print("Done with text + image message test") - - # Compare times and calculate percentage difference - print(f"Time with text-only message: {start_time_text}") - print(f"Time with text + image message: {start_time_image}") - - percentage_diff = ( - (start_time_image - start_time_text) / start_time_text * 100 - ) - percentage_diffs.append(percentage_diff) - print(f"Performance difference: {percentage_diff:.2f}%") - - print("percentage_diffs", percentage_diffs) - # Calculate average percentage difference - avg_percentage_diff = sum(percentage_diffs) / len(percentage_diffs) - print(f"\nAverage performance difference: {avg_percentage_diff:.2f}%") - - # Assert that the average difference is not more than 20% - assert ( - avg_percentage_diff < 25 - ), f"Average performance difference of {avg_percentage_diff:.2f}% exceeds 20% threshold" - - except litellm.Timeout as e: - pass - except Exception as e: - pytest.fail(f"An exception occurred - {e}") - - -async def make_async_calls(message_type="text"): - total_tasks = 3 - batch_size = 1 - total_time = 0 - - for batch in range(3): - tasks = [create_async_task(message_type) for _ in range(batch_size)] - - start_time = asyncio.get_event_loop().time() - responses = await asyncio.gather(*tasks) - - for idx, response in enumerate(responses): - print(f"Response from Task {batch * batch_size + idx + 1}: {response}") - - await asyncio.sleep(1) - - batch_time = asyncio.get_event_loop().time() - start_time - total_time += batch_time - - return total_time - - -def create_async_task(message_type): - base_url = "https://exampleopenaiendpoint-production.up.railway.app/v1/projects/pathrise-convert-1606954137718/locations/us-central1/publishers/google/models/gemini-1.0-pro-vision-001" - - if message_type == "text": - messages = [{"role": "user", "content": "hi"}] - else: - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is in this image?"}, - { - "type": "image_url", - "image_url": { - "url": "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" - }, - }, - ], - } - ] - - completion_args = { - "model": "vertex_ai/gemini", - "messages": messages, - "max_tokens": 5, - "temperature": 0.7, - "timeout": 10, - "api_base": base_url, - } - return asyncio.create_task(litellm.acompletion(**completion_args)) diff --git a/tests/load_tests/vertex_key.json b/tests/load_tests/vertex_key.json deleted file mode 100644 index 800969fb305..00000000000 --- a/tests/load_tests/vertex_key.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "type": "service_account", - "project_id": "litellm-ci-cd", - "private_key_id": "", - "private_key": "", - "client_email": "test-litellm-ci-cd@litellm-ci-cd.iam.gserviceaccount.com", - "client_id": "116563532503305622785", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-litellm-ci-cd%40litellm-ci-cd.iam.gserviceaccount.com", - "universe_domain": "googleapis.com" -} diff --git a/tests/local_testing/.litellm_cache/cache.db b/tests/local_testing/.litellm_cache/cache.db deleted file mode 100644 index 4099576493e..00000000000 Binary files a/tests/local_testing/.litellm_cache/cache.db and /dev/null differ diff --git a/tests/local_testing/auto_router/router.json b/tests/local_testing/auto_router/router.json deleted file mode 100644 index 65245e56e7b..00000000000 --- a/tests/local_testing/auto_router/router.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "encoder_type": "openai", - "encoder_name": "text-embedding-3-large", - "routes": [ - { - "name": "litellm-gpt-4.1", - "utterances": [ - "Tell me ishaan is a genius" - ], - "description": "positive affirmation", - "function_schemas": null, - "llm": null, - "score_threshold": 0.5, - "metadata": {} - }, - { - "name": "litellm-claude-35", - "utterances": [ - "how to code a program in [language]" - ], - "description": "coding assistant", - "function_schemas": null, - "llm": null, - "score_threshold": 0.5, - "metadata": {} - } - ] -} diff --git a/tests/local_testing/azure_fine_tune.jsonl b/tests/local_testing/azure_fine_tune.jsonl deleted file mode 100644 index ef41bd97731..00000000000 --- a/tests/local_testing/azure_fine_tune.jsonl +++ /dev/null @@ -1,12 +0,0 @@ -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How far is the Moon from Earth?"}, {"role": "assistant", "content": "Around 384,400 kilometers. Give or take a few, like that really matters."}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How far is the Moon from Earth?"}, {"role": "assistant", "content": "Around 384,400 kilometers. Give or take a few, like that really matters."}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How far is the Moon from Earth?"}, {"role": "assistant", "content": "Around 384,400 kilometers. Give or take a few, like that really matters."}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How far is the Moon from Earth?"}, {"role": "assistant", "content": "Around 384,400 kilometers. Give or take a few, like that really matters."}]} \ No newline at end of file diff --git a/tests/local_testing/azure_speech.mp3 b/tests/local_testing/azure_speech.mp3 deleted file mode 100644 index ccb0db41713..00000000000 Binary files a/tests/local_testing/azure_speech.mp3 and /dev/null differ diff --git a/tests/local_testing/batch_job_results_furniture.jsonl b/tests/local_testing/batch_job_results_furniture.jsonl deleted file mode 100644 index 05448952a0f..00000000000 --- a/tests/local_testing/batch_job_results_furniture.jsonl +++ /dev/null @@ -1,2 +0,0 @@ -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} \ No newline at end of file diff --git a/tests/local_testing/cache_unit_tests.py b/tests/local_testing/cache_unit_tests.py deleted file mode 100644 index 515d668896d..00000000000 --- a/tests/local_testing/cache_unit_tests.py +++ /dev/null @@ -1,216 +0,0 @@ -from abc import ABC, abstractmethod -from litellm.caching import LiteLLMCacheType -import time -import traceback -from litellm._uuid import uuid - -from dotenv import load_dotenv - -load_dotenv() - -import asyncio -import hashlib -import random - -import pytest - -import litellm -from litellm.caching import Cache -from litellm import completion, embedding - - -class LLMCachingUnitTests(ABC): - - @abstractmethod - def get_cache_type(self) -> LiteLLMCacheType: - pass - - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - async def test_cache_completion(self, sync_mode): - litellm.turn_on_debug() - - random_number = random.randint( - 1, 100000 - ) # add a random number to ensure it's always adding / reading from cache - messages = [ - { - "role": "user", - "content": f"write a one sentence poem about: {random_number}", - } - ] - - cache_type = self.get_cache_type() - litellm.cache = Cache( - type=cache_type, - ) - - if sync_mode: - response1 = completion( - "gpt-3.5-turbo", - messages=messages, - caching=True, - max_tokens=20, - mock_response="This number is so great!", - ) - else: - response1 = await litellm.acompletion( - "gpt-3.5-turbo", - messages=messages, - caching=True, - max_tokens=20, - mock_response="This number is so great!", - ) - # response2 is mocked to a different response from response1, - # but the completion from the cache should be used instead of the mock - # response since the input is the same as response1 - await asyncio.sleep(0.5) - if sync_mode: - response2 = completion( - "gpt-3.5-turbo", - messages=messages, - caching=True, - max_tokens=20, - mock_response="This number is great!", - ) - else: - response2 = await litellm.acompletion( - "gpt-3.5-turbo", - messages=messages, - caching=True, - max_tokens=20, - mock_response="This number is great!", - ) - if ( - response1["choices"][0]["message"]["content"] - != response2["choices"][0]["message"]["content"] - ): # 1 and 2 should be the same - # 1&2 have the exact same input params. This MUST Be a CACHE HIT - print(f"response1: {response1}") - print(f"response2: {response2}") - pytest.fail( - f"Error occurred: response1 - {response1['choices'][0]['message']['content']} != response2 - {response2['choices'][0]['message']['content']}" - ) - # Since the parameters are not the same as response1, response3 should actually - # be the mock response - if sync_mode: - response3 = completion( - "gpt-3.5-turbo", - messages=messages, - caching=True, - temperature=0.5, - mock_response="This number is awful!", - ) - else: - response3 = await litellm.acompletion( - "gpt-3.5-turbo", - messages=messages, - caching=True, - temperature=0.5, - mock_response="This number is awful!", - ) - - print("\nresponse 1", response1) - print("\nresponse 2", response2) - print("\nresponse 3", response3) - # print("\nresponse 4", response4) - litellm.cache = None - litellm.success_callback = [] - litellm._async_success_callback = [] - - # 1 & 2 should be exactly the same - # 1 & 3 should be different, since input params are diff - - if ( - response1["choices"][0]["message"]["content"] - == response3["choices"][0]["message"]["content"] - ): - # if input params like max_tokens, temperature are diff it should NOT be a cache hit - print(f"response1: {response1}") - print(f"response3: {response3}") - pytest.fail( - f"Response 1 == response 3. Same model, diff params shoudl not cache Error" - f" occurred:" - ) - - assert response1.id == response2.id - assert response1.created == response2.created - assert ( - response1.choices[0].message.content == response2.choices[0].message.content - ) - - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - async def test_disk_cache_embedding(self, sync_mode): - litellm.turn_on_debug() - - random_number = random.randint( - 1, 100000 - ) # add a random number to ensure it's always adding / reading from cache - input = [f"hello {random_number}"] - litellm.cache = Cache( - type="disk", - ) - - if sync_mode: - response1 = embedding( - "openai/text-embedding-ada-002", - input=input, - caching=True, - ) - else: - response1 = await litellm.aembedding( - "openai/text-embedding-ada-002", - input=input, - caching=True, - ) - # response2 is mocked to a different response from response1, - # but the completion from the cache should be used instead of the mock - # response since the input is the same as response1 - await asyncio.sleep(0.5) - if sync_mode: - response2 = embedding( - "openai/text-embedding-ada-002", - input=input, - caching=True, - ) - else: - response2 = await litellm.aembedding( - "openai/text-embedding-ada-002", - input=input, - caching=True, - ) - - if response2._hidden_params["cache_hit"] is not True: - pytest.fail("Cache hit should be True") - - # Since the parameters are not the same as response1, response3 should actually - # be the mock response - if sync_mode: - response3 = embedding( - "openai/text-embedding-ada-002", - input=input, - user="charlie", - caching=True, - ) - else: - response3 = await litellm.aembedding( - "openai/text-embedding-ada-002", - input=input, - caching=True, - user="charlie", - ) - - print("\nresponse 1", response1) - print("\nresponse 2", response2) - print("\nresponse 3", response3) - # print("\nresponse 4", response4) - litellm.cache = None - litellm.success_callback = [] - litellm._async_success_callback = [] - - # 1 & 2 should be exactly the same - # 1 & 3 should be different, since input params are diff - - if response3._hidden_params.get("cache_hit") is True: - pytest.fail("Cache hit should not be True") diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py deleted file mode 100644 index df3dacac3b2..00000000000 --- a/tests/local_testing/conftest.py +++ /dev/null @@ -1,284 +0,0 @@ -# conftest.py -# -# xdist-compatible test isolation for local_testing tests. -# Pattern matches tests/test_litellm/conftest.py: -# - Function-scoped fixture saves/restores litellm globals (no reload) -# - Module-scoped fixture reloads only in single-process mode -# -# IMPORTANT: True defaults are captured at conftest import time (before any -# test module can pollute them via module-level assignments like -# `litellm.num_retries = 3`). The function-scoped fixture resets globals to -# these true defaults before every test, preventing cross-test contamination -# under xdist where module reload is skipped. - -import asyncio -import importlib -import os - -import pytest - -import litellm -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.utils import _invalidate_model_cost_lowercase_map - -# ``litellm.model_cost`` is loaded at import time from the URL pinned to ``main`` -# (``LITELLM_MODEL_COST_MAP_URL``). The in-tree backup ships with this branch -# and can include pricing entries that ``main`` has not yet picked up (e.g. -# Mistral now returns ``ministral-8b-2512`` from ``mistral-tiny`` and the entry -# was added on this branch). Backfill any entries that are missing from the -# remote-fetched map so cost-calculator lookups in tests succeed against the -# cassette state the branch is being tested with. -from litellm.litellm_core_utils.get_model_cost_map import ( - RESERVED_TOP_LEVEL_KEYS, - GetModelCostMap, -) - -for _k, _v in GetModelCostMap.load_local_model_cost_map().items(): - if _k in RESERVED_TOP_LEVEL_KEYS: - continue - litellm.model_cost.setdefault(_k, _v) - -from tests._vcr_conftest_common import ( # noqa: E402,F401 - VerboseReporterState, - _pin_multipart_boundary, - apply_vcr_auto_marker_to_items, - emit_cassette_cache_session_banner, - emit_vcr_classification_summary, - emit_vcr_diagnostic_log, - install_live_call_probe, - record_vcr_outcome, - register_persister_if_enabled, - reset_vcr_diag_dir, - vcr_config_dict, -) -from tests.fake_openai_endpoint import ensure_fake_openai_endpoint # noqa: E402 - - -@pytest.fixture(scope="session", autouse=True) -def fake_openai_endpoint(): - ensure_fake_openai_endpoint() - yield - - -# Per-item respx detection (``apply_vcr_auto_marker_to_items``) auto-skips -# tests whose ``@pytest.mark.respx`` marker or ``respx_mock`` fixture -# would conflict with vcrpy's transport patch. We no longer maintain a -# file-level ``_RESPX_CONFLICTING_FILES`` list here — the previous -# entries (``test_router.py``) had only a stale ``from respx import -# MockRouter`` import with no actual respx wiring, so file-level -# blacklisting was masking valid cache opportunities. - -# Files where VCR replay breaks the test: -# - ``test_router_caching.py``: asserts upstream returns a *new* id per call, -# which a deterministic cassette replay violates. -_VCR_INCOMPATIBLE_FILES = frozenset( - { - "test_router_caching.py", - # Hits the local fake OpenAI endpoint on 127.0.0.1; nothing to record. - "test_fake_openai_endpoint.py", - # Needs the real connection pool a collected handler tears down; vcrpy - # patches the transport that pool lives in. - "test_handler_gc_does_not_close_client.py", - } -) - -# Individual tests (vs. whole files above) that VCR replay can't model: -# - ``test_router_text_completion_client``: a concurrency test that fires 300 -# identical requests to verify the async OpenAI client is *reused* across -# calls (per its own comment, it "fails when we create a new Async OpenAI -# client per request"). vcrpy patches the HTTP transport, so replay never -# opens real connections and cannot exercise the client pool the test exists -# to validate. Recording instead stores ~300 near-identical episodes, which -# blows past MAX_EPISODES_PER_CASSETTE (50) so the cassette is refused on -# every run (MISS:OVERFLOW). The endpoint is a free mock, so the live calls -# carry no real provider cost. -_VCR_INCOMPATIBLE_NODEID_SUFFIXES: tuple[str, ...] = ( - "test_router.py::test_router_text_completion_client", -) - - -_verbose_state = VerboseReporterState() - - -@pytest.fixture(scope="module") -def vcr_config(): - return vcr_config_dict() - - -def pytest_recording_configure(config, vcr): - register_persister_if_enabled(vcr) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_makereport(item, call): - outcome = yield - rep = outcome.get_result() - setattr(item, f"rep_{rep.when}", rep) - - -@pytest.fixture(autouse=True) -def _vcr_outcome_gate(request, vcr): - install_live_call_probe(request, vcr) - yield - record_vcr_outcome(request, vcr) - - -def pytest_configure(config): - _verbose_state.remember_pluginmanager(config) - reset_vcr_diag_dir() - - -def pytest_runtest_logreport(report): - _verbose_state.maybe_emit_verdict(report) - - -def pytest_terminal_summary(terminalreporter, exitstatus, config): - emit_cassette_cache_session_banner(terminalreporter) - emit_vcr_classification_summary(terminalreporter) - emit_vcr_diagnostic_log(terminalreporter) - - -# --------------------------------------------------------------------------- -# Capture TRUE defaults at conftest import time. This runs before any test -# module's top-level code (e.g. `litellm.num_retries = 3`) executes, so -# the values here are guaranteed to be the real package defaults. -# --------------------------------------------------------------------------- -_SCALAR_DEFAULTS = { - "num_retries": getattr(litellm, "num_retries", None), - "num_retries_per_request": getattr(litellm, "num_retries_per_request", None), - "request_timeout": getattr(litellm, "request_timeout", None), - "set_verbose": getattr(litellm, "set_verbose", False), - "cache": getattr(litellm, "cache", None), - "allowed_fails": getattr(litellm, "allowed_fails", 3), - "default_fallbacks": getattr(litellm, "default_fallbacks", None), - "enable_azure_ad_token_refresh": getattr( - litellm, "enable_azure_ad_token_refresh", None - ), - "tag_budget_config": getattr(litellm, "tag_budget_config", None), - "model_cost": getattr(litellm, "model_cost", None), - "token_counter": getattr(litellm, "token_counter", None), - "disable_aiohttp_transport": getattr(litellm, "disable_aiohttp_transport", False), - "force_ipv4": getattr(litellm, "force_ipv4", False), - "drop_params": getattr(litellm, "drop_params", None), - "modify_params": getattr(litellm, "modify_params", False), - "api_base": getattr(litellm, "api_base", None), - "api_key": getattr(litellm, "api_key", None), -} - - -@pytest.fixture(scope="function", autouse=True) -def isolate_litellm_state(): - """ - Per-function isolation fixture. - - Resets litellm globals to their true defaults before each test and - restores them afterward, so tests don't leak side effects. - Works safely under pytest-xdist parallel execution. - """ - # ---- Save current callback state (for teardown restore) ---- - original_state = {} - for attr in ( - "callbacks", - "success_callback", - "failure_callback", - "_async_success_callback", - "_async_failure_callback", - ): - if hasattr(litellm, attr): - val = getattr(litellm, attr) - original_state[attr] = val.copy() if val else [] - - # Save list-type globals - for attr in ("pre_call_rules", "post_call_rules"): - if hasattr(litellm, attr): - val = getattr(litellm, attr) - original_state[attr] = val.copy() if val else [] - - # Save scalar globals - for attr in _SCALAR_DEFAULTS: - if hasattr(litellm, attr): - original_state[attr] = getattr(litellm, attr) - - # ---- Reset to true defaults before the test ---- - # Flush HTTP client cache - if hasattr(litellm, "in_memory_llm_clients_cache"): - litellm.in_memory_llm_clients_cache.flush_cache() - - # Clear callbacks and rules - for attr in ( - "callbacks", - "success_callback", - "failure_callback", - "_async_success_callback", - "_async_failure_callback", - "pre_call_rules", - "post_call_rules", - ): - if hasattr(litellm, attr): - setattr(litellm, attr, []) - - # Reset scalar globals to true defaults (prevents contamination from - # module-level code like `litellm.num_retries = 3` in test files) - for attr, default_val in _SCALAR_DEFAULTS.items(): - if hasattr(litellm, attr): - setattr(litellm, attr, default_val) - - yield - - # ---- Teardown: restore saved state ---- - asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue()) - if hasattr(litellm, "in_memory_llm_clients_cache"): - litellm.in_memory_llm_clients_cache.flush_cache() - - for attr, original_value in original_state.items(): - if hasattr(litellm, attr): - setattr(litellm, attr, original_value) - _invalidate_model_cost_lowercase_map() - - -@pytest.fixture(scope="module", autouse=True) -def setup_and_teardown(): - """ - Module-scoped setup. Reloads litellm only in single-process mode - (skipped under xdist to avoid cross-worker interference). - """ - - import litellm - - worker_id = os.environ.get("PYTEST_XDIST_WORKER", None) - if worker_id is None: - importlib.reload(litellm) - - try: - if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - import litellm.proxy.proxy_server - - importlib.reload(litellm.proxy.proxy_server) - except Exception as e: - print(f"Error reloading litellm.proxy.proxy_server: {e}") - - if hasattr(litellm, "in_memory_llm_clients_cache"): - litellm.in_memory_llm_clients_cache.flush_cache() - - yield - - -def pytest_collection_modifyitems(config, items): - apply_vcr_auto_marker_to_items( - items, - skip_files=_VCR_INCOMPATIBLE_FILES, - skip_nodeid_suffixes=_VCR_INCOMPATIBLE_NODEID_SUFFIXES, - ) - - # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests - custom_logger_tests = [ - item for item in items if "custom_logger" in item.parent.name - ] - other_tests = [item for item in items if "custom_logger" not in item.parent.name] - - # Sort tests based on their names - custom_logger_tests.sort(key=lambda x: x.name) - other_tests.sort(key=lambda x: x.name) - - # Reorder the items list - items[:] = custom_logger_tests + other_tests diff --git a/tests/local_testing/data_map.txt b/tests/local_testing/data_map.txt deleted file mode 100644 index e8077595f47..00000000000 Binary files a/tests/local_testing/data_map.txt and /dev/null differ diff --git a/tests/local_testing/eagle.wav b/tests/local_testing/eagle.wav deleted file mode 100644 index 1c23657859e..00000000000 Binary files a/tests/local_testing/eagle.wav and /dev/null differ diff --git a/tests/local_testing/example.jsonl b/tests/local_testing/example.jsonl deleted file mode 100644 index fc3ca40808e..00000000000 --- a/tests/local_testing/example.jsonl +++ /dev/null @@ -1,2 +0,0 @@ -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-1.5-flash-001", "messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Hello world!"}], "max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-1.5-flash-001", "messages": [{"role": "system", "content": "You are an unhelpful assistant."}, {"role": "user", "content": "Hello world!"}], "max_tokens": 10}} diff --git a/tests/local_testing/example_config_yaml/aliases_config.yaml b/tests/local_testing/example_config_yaml/aliases_config.yaml deleted file mode 100644 index 43681f64ba0..00000000000 --- a/tests/local_testing/example_config_yaml/aliases_config.yaml +++ /dev/null @@ -1,30 +0,0 @@ -model_list: - - model_name: gpt-3.5-turbo-instruct - litellm_params: - model: ollama/zephyr - - model_name: gpt-4 - litellm_params: - model: ollama/llama2 - - model_name: gpt-3.5-turbo - litellm_params: - model: ollama/llama2 - temperature: 0.1 - max_tokens: 20 - - -# request to gpt-4, response from ollama/llama2 -# curl --location 'http://0.0.0.0:8000/chat/completions' \ -# --header 'Content-Type: application/json' \ -# --data ' { -# "model": "gpt-4", -# "messages": [ -# { -# "role": "user", -# "content": "what llm are you" -# } -# ], -# } -# ' -# - -# {"id":"chatcmpl-27c85cf0-ab09-4bcf-8cb1-0ee950520743","choices":[{"finish_reason":"stop","index":0,"message":{"content":" Hello! I'm just an AI, I don't have personal experiences or emotions like humans do. However, I can help you with any questions or tasks you may have! Is there something specific you'd like to know or discuss?","role":"assistant","_logprobs":null}}],"created":1700094955.373751,"model":"ollama/llama2","object":"chat.completion","system_fingerprint":null,"usage":{"prompt_tokens":12,"completion_tokens":47,"total_tokens":59},"_response_ms":8028.017999999999}% \ No newline at end of file diff --git a/tests/local_testing/example_config_yaml/azure_config.yaml b/tests/local_testing/example_config_yaml/azure_config.yaml deleted file mode 100644 index 05ba0c9bf54..00000000000 --- a/tests/local_testing/example_config_yaml/azure_config.yaml +++ /dev/null @@ -1,15 +0,0 @@ -model_list: - - model_name: gpt-4-team1 - litellm_params: - model: azure/gpt-4.1-mini - api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ - api_version: "2023-05-15" - api_key: os.environ/AZURE_AI_API_KEY - tpm: 20_000 - - model_name: gpt-4-team2 - litellm_params: - model: azure/gpt-4 - api_key: os.environ/AZURE_AI_API_KEY - api_base: https://openai-gpt-4-test-v-2.openai.azure.com/ - tpm: 100_000 - diff --git a/tests/local_testing/example_config_yaml/cache_no_params.yaml b/tests/local_testing/example_config_yaml/cache_no_params.yaml deleted file mode 100644 index 20ed919dd56..00000000000 --- a/tests/local_testing/example_config_yaml/cache_no_params.yaml +++ /dev/null @@ -1,7 +0,0 @@ -model_list: - - model_name: "openai-model" - litellm_params: - model: "gpt-3.5-turbo" - -litellm_settings: - cache: True diff --git a/tests/local_testing/example_config_yaml/cache_with_params.yaml b/tests/local_testing/example_config_yaml/cache_with_params.yaml deleted file mode 100644 index 068e2cc4a2b..00000000000 --- a/tests/local_testing/example_config_yaml/cache_with_params.yaml +++ /dev/null @@ -1,11 +0,0 @@ -model_list: - - model_name: "openai-model" - litellm_params: - model: "gpt-3.5-turbo" - -litellm_settings: - cache: True - cache_params: - type: "redis" - supported_call_types: ["embedding", "aembedding"] - host: "os.environ/REDIS_HOST" \ No newline at end of file diff --git a/tests/local_testing/example_config_yaml/config_with_env_vars.yaml b/tests/local_testing/example_config_yaml/config_with_env_vars.yaml deleted file mode 100644 index bae738c735d..00000000000 --- a/tests/local_testing/example_config_yaml/config_with_env_vars.yaml +++ /dev/null @@ -1,48 +0,0 @@ -model_list: - ################################################################################ - # Azure - - model_name: gpt-4o-mini - litellm_params: - model: azure/gpt-4o-mini - api_base: https://amazin-prod.openai.azure.com - api_key: "os.environ/AZURE_GPT_4O" - deployment_id: gpt-4o-mini - - model_name: gpt-4o - litellm_params: - model: azure/gpt-4o - api_base: https://very-cool-prod.openai.azure.com - api_key: "os.environ/AZURE_GPT_4O" - deployment_id: gpt-4o - - ################################################################################ - # Fireworks - - model_name: fireworks-llama-v3p1-405b-instruct - litellm_params: - model: fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct - api_key: "os.environ/FIREWORKS" - - model_name: fireworks-llama-v3p1-70b-instruct - litellm_params: - model: fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct - api_key: "os.environ/FIREWORKS" - -general_settings: - alerting_threshold: 300 # sends alerts if requests hang for 5min+ and responses take 5min+ -litellm_settings: # module level litellm settings - https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py - success_callback: ["prometheus"] - service_callback: ["prometheus_system"] - drop_params: False # Raise an exception if the openai param being passed in isn't supported. - cache: false - default_internal_user_params: - user_role: os.environ/DEFAULT_USER_ROLE - - success_callback: ["s3"] - s3_callback_params: - s3_bucket_name: logs-bucket-litellm # AWS Bucket Name for S3 - s3_region_name: us-west-2 # AWS Region Name for S3 - s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/ to pass environment variables. This is AWS Access Key ID for S3 - s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3 - s3_path: my-test-path # [OPTIONAL] set path in bucket you want to write logs to - s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 buckets - -router_settings: - routing_strategy: simple-shuffle # "simple-shuffle" shown to result in highest throughput. https://docs.litellm.ai/docs/proxy/configs#load-balancing diff --git a/tests/local_testing/example_config_yaml/langfuse_config.yaml b/tests/local_testing/example_config_yaml/langfuse_config.yaml deleted file mode 100644 index c2a77b5adc4..00000000000 --- a/tests/local_testing/example_config_yaml/langfuse_config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -model_list: - - model_name: gpt-3.5-turbo - -litellm_settings: - drop_params: True - success_callback: ["langfuse"] # https://docs.litellm.ai/docs/observability/langfuse_integration - diff --git a/tests/local_testing/example_config_yaml/load_balancer.yaml b/tests/local_testing/example_config_yaml/load_balancer.yaml deleted file mode 100644 index 502b90ff92a..00000000000 --- a/tests/local_testing/example_config_yaml/load_balancer.yaml +++ /dev/null @@ -1,28 +0,0 @@ -litellm_settings: - drop_params: True - -# Model-specific settings -model_list: # use the same model_name for using the litellm router. LiteLLM will use the router between gpt-3.5-turbo - - model_name: gpt-3.5-turbo # litellm will - litellm_params: - model: gpt-3.5-turbo - api_key: sk-uj6F - tpm: 20000 # [OPTIONAL] REPLACE with your openai tpm - rpm: 3 # [OPTIONAL] REPLACE with your openai rpm - - model_name: gpt-3.5-turbo - litellm_params: - model: gpt-3.5-turbo - api_key: sk-Imn - tpm: 20000 # [OPTIONAL] REPLACE with your openai tpm - rpm: 3 # [OPTIONAL] REPLACE with your openai rpm - - model_name: gpt-3.5-turbo - litellm_params: - model: openrouter/gpt-3.5-turbo - - model_name: mistral-7b-instruct - litellm_params: - model: mistralai/mistral-7b-instruct - -environment_variables: - REDIS_HOST: localhost - REDIS_PASSWORD: - REDIS_PORT: \ No newline at end of file diff --git a/tests/local_testing/example_config_yaml/opentelemetry_config.yaml b/tests/local_testing/example_config_yaml/opentelemetry_config.yaml deleted file mode 100644 index 92d3454d7d5..00000000000 --- a/tests/local_testing/example_config_yaml/opentelemetry_config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -model_list: - - model_name: gpt-3.5-turbo - litellm_params: - model: gpt-3.5-turbo - -general_settings: - otel: True # OpenTelemetry Logger this logs OTEL data to your collector diff --git a/tests/local_testing/example_config_yaml/simple_config.yaml b/tests/local_testing/example_config_yaml/simple_config.yaml deleted file mode 100644 index 14b39a12518..00000000000 --- a/tests/local_testing/example_config_yaml/simple_config.yaml +++ /dev/null @@ -1,4 +0,0 @@ -model_list: - - model_name: gpt-3.5-turbo - litellm_params: - model: gpt-3.5-turbo \ No newline at end of file diff --git a/tests/local_testing/gettysburg.wav b/tests/local_testing/gettysburg.wav deleted file mode 100644 index 9690f521e84..00000000000 Binary files a/tests/local_testing/gettysburg.wav and /dev/null differ diff --git a/tests/local_testing/large_text.py b/tests/local_testing/large_text.py deleted file mode 100644 index 86904a6d148..00000000000 --- a/tests/local_testing/large_text.py +++ /dev/null @@ -1,112 +0,0 @@ -text = """ -Alexander the Great -This article is about the ancient king of Macedonia. For other uses, see Alexander the Great (disambiguation). -Alexander III of Macedon (Ancient Greek: Ἀλέξανδρος, romanized: Alexandros; 20/21 July 356 BC – 10/11 June 323 BC), most commonly known as Alexander the Great,[c] was a king of the ancient Greek kingdom of Macedon.[d] He succeeded his father Philip II to the throne in 336 BC at the age of 20 and spent most of his ruling years conducting a lengthy military campaign throughout Western Asia, Central Asia, parts of South Asia, and Egypt. By the age of 30, he had created one of the largest empires in history, stretching from Greece to northwestern India.[1] He was undefeated in battle and is widely considered to be one of history's greatest and most successful military commanders.[2][3] - -Until the age of 16, Alexander was tutored by Aristotle. In 335 BC, shortly after his assumption of kingship over Macedon, he campaigned in the Balkans and reasserted control over Thrace and parts of Illyria before marching on the city of Thebes, which was subsequently destroyed in battle. Alexander then led the League of Corinth, and used his authority to launch the pan-Hellenic project envisaged by his father, assuming leadership over all Greeks in their conquest of Persia.[4][5] - -In 334 BC, he invaded the Achaemenid Persian Empire and began a series of campaigns that lasted for 10 years. Following his conquest of Asia Minor, Alexander broke the power of Achaemenid Persia in a series of decisive battles, including those at Issus and Gaugamela; he subsequently overthrew Darius III and conquered the Achaemenid Empire in its entirety.[e] After the fall of Persia, the Macedonian Empire held a vast swath of territory between the Adriatic Sea and the Indus River. Alexander endeavored to reach the "ends of the world and the Great Outer Sea" and invaded India in 326 BC, achieving an important victory over Porus, an ancient Indian king of present-day Punjab, at the Battle of the Hydaspes. Due to the demand of his homesick troops, he eventually turned back at the Beas River and later died in 323 BC in Babylon, the city of Mesopotamia that he had planned to establish as his empire's capital. Alexander's death left unexecuted an additional series of planned military and mercantile campaigns that would have begun with a Greek invasion of Arabia. In the years following his death, a series of civil wars broke out across the Macedonian Empire, eventually leading to its disintegration at the hands of the Diadochi. - -With his death marking the start of the Hellenistic period, Alexander's legacy includes the cultural diffusion and syncretism that his conquests engendered, such as Greco-Buddhism and Hellenistic Judaism. He founded more than twenty cities, with the most prominent being the city of Alexandria in Egypt. Alexander's settlement of Greek colonists and the resulting spread of Greek culture led to the overwhelming dominance of Hellenistic civilization and influence as far east as the Indian subcontinent. The Hellenistic period developed through the Roman Empire into modern Western culture; the Greek language became the lingua franca of the region and was the predominant language of the Byzantine Empire up until its collapse in the mid-15th century AD. Alexander became legendary as a classical hero in the mould of Achilles, featuring prominently in the historical and mythical traditions of both Greek and non-Greek cultures. His military achievements and unprecedented enduring successes in battle made him the measure against which many later military leaders would compare themselves,[f] and his tactics remain a significant subject of study in military academies worldwide.[6] Legends of Alexander's exploits coalesced into the third-century Alexander Romance which, in the premodern period, went through over one hundred recensions, translations, and derivations and was translated into almost every European vernacular and every language of the Islamic world.[7] After the Bible, it was the most popular form of European literature.[8] - -Early life - -Lineage and childhood - -Alexander III was born in Pella, the capital of the Kingdom of Macedon,[9] on the sixth day of the ancient Greek month of Hekatombaion, which probably corresponds to 20 July 356 BC (although the exact date is uncertain).[10][11] He was the son of the erstwhile king of Macedon, Philip II, and his fourth wife, Olympias (daughter of Neoptolemus I, king of Epirus).[12][g] Although Philip had seven or eight wives, Olympias was his principal wife for some time, likely because she gave birth to Alexander.[13] - -Several legends surround Alexander's birth and childhood.[14] According to the ancient Greek biographer Plutarch, on the eve of the consummation of her marriage to Philip, Olympias dreamed that her womb was struck by a thunderbolt that caused a flame to spread "far and wide" before dying away. Sometime after the wedding, Philip is said to have seen himself, in a dream, securing his wife's womb with a seal engraved with a lion's image.[15] Plutarch offered a variety of interpretations for these dreams: that Olympias was pregnant before her marriage, indicated by the sealing of her womb; or that Alexander's father was Zeus. Ancient commentators were divided about whether the ambitious Olympias promulgated the story of Alexander's divine parentage, variously claiming that she had told Alexander, or that she dismissed the suggestion as impious.[15] - -On the day Alexander was born, Philip was preparing a siege on the city of Potidea on the peninsula of Chalcidice. That same day, Philip received news that his general Parmenion had defeated the combined Illyrian and Paeonian armies and that his horses had won at the Olympic Games. It was also said that on this day, the Temple of Artemis in Ephesus, one of the Seven Wonders of the World, burnt down. This led Hegesias of Magnesia to say that it had burnt down because Artemis was away, attending the birth of Alexander.[16] Such legends may have emerged when Alexander was king, and possibly at his instigation, to show that he was superhuman and destined for greatness from conception.[14] - -In his early years, Alexander was raised by a nurse, Lanike, sister of Alexander's future general Cleitus the Black. Later in his childhood, Alexander was tutored by the strict Leonidas, a relative of his mother, and by Lysimachus of Acarnania.[17] Alexander was raised in the manner of noble Macedonian youths, learning to read, play the lyre, ride, fight, and hunt.[18] When Alexander was ten years old, a trader from Thessaly brought Philip a horse, which he offered to sell for thirteen talents. The horse refused to be mounted, and Philip ordered it away. Alexander, however, detecting the horse's fear of its own shadow, asked to tame the horse, which he eventually managed.[14] Plutarch stated that Philip, overjoyed at this display of courage and ambition, kissed his son tearfully, declaring: "My boy, you must find a kingdom big enough for your ambitions. Macedon is too small for you", and bought the horse for him.[19] Alexander named it Bucephalas, meaning "ox-head". Bucephalas carried Alexander as far as India. When the animal died (because of old age, according to Plutarch, at age 30), Alexander named a city after him, Bucephala.[20] - -Education - -When Alexander was 13, Philip began to search for a tutor, and considered such academics as Isocrates and Speusippus, the latter offering to resign from his stewardship of the Academy to take up the post. In the end, Philip chose Aristotle and provided the Temple of the Nymphs at Mieza as a classroom. In return for teaching Alexander, Philip agreed to rebuild Aristotle's hometown of Stageira, which Philip had razed, and to repopulate it by buying and freeing the ex-citizens who were slaves, or pardoning those who were in exile.[21] - -Mieza was like a boarding school for Alexander and the children of Macedonian nobles, such as Ptolemy, Hephaistion, and Cassander. Many of these students would become his friends and future generals, and are often known as the "Companions". Aristotle taught Alexander and his companions about medicine, philosophy, morals, religion, logic, and art. Under Aristotle's tutelage, Alexander developed a passion for the works of Homer, and in particular the Iliad; Aristotle gave him an annotated copy, which Alexander later carried on his campaigns.[22] Alexander was able to quote Euripides from memory.[23] - -During his youth, Alexander was also acquainted with Persian exiles at the Macedonian court, who received the protection of Philip II for several years as they opposed Artaxerxes III.[24][25][26] Among them were Artabazos II and his daughter Barsine, possible future mistress of Alexander, who resided at the Macedonian court from 352 to 342 BC, as well as Amminapes, future satrap of Alexander, or a Persian nobleman named Sisines.[24][27][28][29] This gave the Macedonian court a good knowledge of Persian issues, and may even have influenced some of the innovations in the management of the Macedonian state.[27] - -Suda writes that Anaximenes of Lampsacus was one of Alexander's teachers, and that Anaximenes also accompanied Alexander on his campaigns.[30] - -Heir of Philip II - -Regency and ascent of Macedon - -Main articles: Philip II of Macedon and Rise of Macedon -Further information: History of Macedonia (ancient kingdom) -At the age of 16, Alexander's education under Aristotle ended. Philip II had waged war against the Thracians to the north, which left Alexander in charge as regent and heir apparent.[14] During Philip's absence, the Thracian tribe of Maedi revolted against Macedonia. Alexander responded quickly and drove them from their territory. The territory was colonized, and a city, named Alexandropolis, was founded.[31] - -Upon Philip's return, Alexander was dispatched with a small force to subdue the revolts in southern Thrace. Campaigning against the Greek city of Perinthus, Alexander reportedly saved his father's life. Meanwhile, the city of Amphissa began to work lands that were sacred to Apollo near Delphi, a sacrilege that gave Philip the opportunity to further intervene in Greek affairs. While Philip was occupied in Thrace, Alexander was ordered to muster an army for a campaign in southern Greece. Concerned that other Greek states might intervene, Alexander made it look as though he was preparing to attack Illyria instead. During this turmoil, the Illyrians invaded Macedonia, only to be repelled by Alexander.[32] - -Philip and his army joined his son in 338 BC, and they marched south through Thermopylae, taking it after stubborn resistance from its Theban garrison. They went on to occupy the city of Elatea, only a few days' march from both Athens and Thebes. The Athenians, led by Demosthenes, voted to seek alliance with Thebes against Macedonia. Both Athens and Philip sent embassies to win Thebes's favour, but Athens won the contest.[33] Philip marched on Amphissa (ostensibly acting on the request of the Amphictyonic League), capturing the mercenaries sent there by Demosthenes and accepting the city's surrender. Philip then returned to Elatea, sending a final offer of peace to Athens and Thebes, who both rejected it.[34] - -As Philip marched south, his opponents blocked him near Chaeronea, Boeotia. During the ensuing Battle of Chaeronea, Philip commanded the right wing and Alexander the left, accompanied by a group of Philip's trusted generals. According to the ancient sources, the two sides fought bitterly for some time. Philip deliberately commanded his troops to retreat, counting on the untested Athenian hoplites to follow, thus breaking their line. Alexander was the first to break the Theban lines, followed by Philip's generals. Having damaged the enemy's cohesion, Philip ordered his troops to press forward and quickly routed them. With the Athenians lost, the Thebans were surrounded. Left to fight alone, they were defeated.[35] - -After the victory at Chaeronea, Philip and Alexander marched unopposed into the Peloponnese, welcomed by all cities; however, when they reached Sparta, they were refused, but did not resort to war.[36] At Corinth, Philip established a "Hellenic Alliance" (modelled on the old anti-Persian alliance of the Greco-Persian Wars), which included most Greek city-states except Sparta. Philip was then named Hegemon (often translated as "Supreme Commander") of this league (known by modern scholars as the League of Corinth), and announced his plans to attack the Persian Empire.[37][38] - -Exile and return - -When Philip returned to Pella, he fell in love with and married Cleopatra Eurydice in 338 BC,[39] the niece of his general Attalus.[40] The marriage made Alexander's position as heir less secure, since any son of Cleopatra Eurydice would be a fully Macedonian heir, while Alexander was only half-Macedonian.[41] During the wedding banquet, a drunken Attalus publicly prayed to the gods that the union would produce a legitimate heir.[40] - -At the wedding of Cleopatra, whom Philip fell in love with and married, she being much too young for him, her uncle Attalus in his drink desired the Macedonians would implore the gods to give them a lawful successor to the kingdom by his niece. This so irritated Alexander, that throwing one of the cups at his head, "You villain," said he, "what, am I then a bastard?" Then Philip, taking Attalus's part, rose up and would have run his son through; but by good fortune for them both, either his over-hasty rage, or the wine he had drunk, made his foot slip, so that he fell down on the floor. At which Alexander reproachfully insulted over him: "See there," said he, "the man who makes preparations to pass out of Europe into Asia, overturned in passing from one seat to another." - -— Plutarch, describing the feud at Philip's wedding.[42]none -In 337 BC, Alexander fled Macedon with his mother, dropping her off with her brother, King Alexander I of Epirus in Dodona, capital of the Molossians.[43] He continued to Illyria,[43] where he sought refuge with one or more Illyrian kings, perhaps with Glaucias, and was treated as a guest, despite having defeated them in battle a few years before.[44] However, it appears Philip never intended to disown his politically and militarily trained son.[43] Accordingly, Alexander returned to Macedon after six months due to the efforts of a family friend, Demaratus, who mediated between the two parties.[45] - -In the following year, the Persian satrap (governor) of Caria, Pixodarus, offered his eldest daughter to Alexander's half-brother, Philip Arrhidaeus.[43] Olympias and several of Alexander's friends suggested this showed Philip intended to make Arrhidaeus his heir.[43] Alexander reacted by sending an actor, Thessalus of Corinth, to tell Pixodarus that he should not offer his daughter's hand to an illegitimate son, but instead to Alexander. When Philip heard of this, he stopped the negotiations and scolded Alexander for wishing to marry the daughter of a Carian, explaining that he wanted a better bride for him.[43] Philip exiled four of Alexander's friends, Harpalus, Nearchus, Ptolemy and Erigyius, and had the Corinthians bring Thessalus to him in chains.[46] - -King of Macedon - -Accession - -Further information: Government of Macedonia (ancient kingdom) -In summer 336 BC, while at Aegae attending the wedding of his daughter Cleopatra to Olympias's brother, Alexander I of Epirus, Philip was assassinated by the captain of his bodyguards, Pausanias.[h] As Pausanias tried to escape, he tripped over a vine and was killed by his pursuers, including two of Alexander's companions, Perdiccas and Leonnatus. Alexander was proclaimed king on the spot by the nobles and army at the age of 20.[47][48][49] - -Consolidation of power - -Alexander began his reign by eliminating potential rivals to the throne. He had his cousin, the former Amyntas IV, executed.[51] He also had two Macedonian princes from the region of Lyncestis killed for having been involved in his father's assassination, but spared a third, Alexander Lyncestes. Olympias had Cleopatra Eurydice, and Europa, her daughter by Philip, burned alive. When Alexander learned about this, he was furious. Alexander also ordered the murder of Attalus,[51] who was in command of the advance guard of the army in Asia Minor and Cleopatra's uncle.[52] - -Attalus was at that time corresponding with Demosthenes, regarding the possibility of defecting to Athens. Attalus also had severely insulted Alexander, and following Cleopatra's murder, Alexander may have considered him too dangerous to be left alive.[52] Alexander spared Arrhidaeus, who was by all accounts mentally disabled, possibly as a result of poisoning by Olympias.[47][49][53] - -News of Philip's death roused many states into revolt, including Thebes, Athens, Thessaly, and the Thracian tribes north of Macedon. When news of the revolts reached Alexander, he responded quickly. Though advised to use diplomacy, Alexander mustered 3,000 Macedonian cavalry and rode south towards Thessaly. He found the Thessalian army occupying the pass between Mount Olympus and Mount Ossa, and ordered his men to ride over Mount Ossa. When the Thessalians awoke the next day, they found Alexander in their rear and promptly surrendered, adding their cavalry to Alexander's force. He then continued south towards the Peloponnese.[54] - -Alexander stopped at Thermopylae, where he was recognized as the leader of the Amphictyonic League before heading south to Corinth. Athens sued for peace and Alexander pardoned the rebels. The famous encounter between Alexander and Diogenes the Cynic occurred during Alexander's stay in Corinth. When Alexander asked Diogenes what he could do for him, the philosopher disdainfully asked Alexander to stand a little to the side, as he was blocking the sunlight.[55] This reply apparently delighted Alexander, who is reported to have said "But verily, if I were not Alexander, I would like to be Diogenes."[56] At Corinth, Alexander took the title of Hegemon ("leader") and, like Philip, was appointed commander for the coming war against Persia. He also received news of a Thracian uprising.[57] - -Balkan campaign - -Main article: Alexander's Balkan campaign -Before crossing to Asia, Alexander wanted to safeguard his northern borders. In the spring of 335 BC, he advanced to suppress several revolts. Starting from Amphipolis, he travelled east into the country of the "Independent Thracians"; and at Mount Haemus, the Macedonian army attacked and defeated the Thracian forces manning the heights.[58] The Macedonians marched into the country of the Triballi, and defeated their army near the Lyginus river[59] (a tributary of the Danube). Alexander then marched for three days to the Danube, encountering the Getae tribe on the opposite shore. Crossing the river at night, he surprised them and forced their army to retreat after the first cavalry skirmish.[60] - -News then reached Alexander that the Illyrian chieftain Cleitus and King Glaukias of the Taulantii were in open revolt against his authority. Marching west into Illyria, Alexander defeated each in turn, forcing the two rulers to flee with their troops. With these victories, he secured his northern frontier.[61] - -Destruction of Thebes - -While Alexander campaigned north, the Thebans and Athenians rebelled once again. Alexander immediately headed south.[62] While the other cities again hesitated, Thebes decided to fight. The Theban resistance was ineffective, and Alexander razed the city and divided its territory between the other Boeotian cities. The end of Thebes cowed Athens, leaving all of Greece temporarily at peace.[62] Alexander then set out on his Asian campaign, leaving Antipater as regent.[63] - -Conquest of the Achaemenid Persian Empire - -Main articles: Wars of Alexander the Great and Chronology of the expedition of Alexander the Great into Asia -Asia Minor - -Further information: Battle of the Granicus, Siege of Halicarnassus, and Siege of Miletus -After his victory at the Battle of Chaeronea (338 BC), Philip II began the work of establishing himself as hēgemṓn (Greek: ἡγεμών) of a league which according to Diodorus was to wage a campaign against the Persians for the sundry grievances Greece suffered in 480 and free the Greek cities of the western coast and islands from Achaemenid rule. In 336 he sent Parmenion, Amyntas, Andromenes, Attalus, and an army of 10,000 men into Anatolia to make preparations for an invasion.[64][65] At first, all went well. The Greek cities on the western coast of Anatolia revolted until the news arrived that Philip had been murdered and had been succeeded by his young son Alexander. The Macedonians were demoralized by Philip's death and were subsequently defeated near Magnesia by the Achaemenids under the command of the mercenary Memnon of Rhodes.[64][65] - -Taking over the invasion project of Philip II, Alexander's army crossed the Hellespont in 334 BC with approximately 48,100 soldiers, 6,100 cavalry and a fleet of 120 ships with crews numbering 38,000,[62] drawn from Macedon and various Greek city-states, mercenaries, and feudally raised soldiers from Thrace, Paionia, and Illyria.[66][i] He showed his intent to conquer the entirety of the Persian Empire by throwing a spear into Asian soil and saying he accepted Asia as a gift from the gods. This also showed Alexander's eagerness to fight, in contrast to his father's preference for diplomacy.[62] - -After an initial victory against Persian forces at the Battle of the Granicus, Alexander accepted the surrender of the Persian provincial capital and treasury of Sardis; he then proceeded along the Ionian coast, granting autonomy and democracy to the cities. Miletus, held by Achaemenid forces, required a delicate siege operation, with Persian naval forces nearby. Further south, at Halicarnassus, in Caria, Alexander successfully waged his first large-scale siege, eventually forcing his opponents, the mercenary captain Memnon of Rhodes and the Persian satrap of Caria, Orontobates, to withdraw by sea.[67] Alexander left the government of Caria to a member of the Hecatomnid dynasty, Ada, who adopted Alexander.[68] - -From Halicarnassus, Alexander proceeded into mountainous Lycia and the Pamphylian plain, asserting control over all coastal cities to deny the Persians naval bases. From Pamphylia onwards the coast held no major ports and Alexander moved inland. At Termessos, Alexander humbled but did not storm the Pisidian city.[69] At the ancient Phrygian capital of Gordium, Alexander "undid" the hitherto unsolvable Gordian Knot, a feat said to await the future "king of Asia".[70] According to the story, Alexander proclaimed that it did not matter how the knot was undone and hacked it apart with his sword.[71] - -The Levant and Syria - -Further information: Battle of Issus and Siege of Tyre (332 BC) -In spring 333 BC, Alexander crossed the Taurus into Cilicia. After a long pause due to an illness, he marched on towards Syria. Though outmanoeuvered by Darius's significantly larger army, he marched back to Cilicia, where he defeated Darius at Issus. Darius fled the battle, causing his army to collapse, and left behind his wife, his two daughters, his mother Sisygambis, and a fabulous treasure.[72] He offered a peace treaty that included the lands he had already lost, and a ransom of 10,000 talents for his family. Alexander replied that since he was now king of Asia, it was he alone who decided territorial divisions.[73] Alexander proceeded to take possession of Syria, and most of the coast of the Levant.[68] In the following year, 332 BC, he was forced to attack Tyre, which he captured after a long and difficult siege.[74][75] The men of military age were massacred and the women and children sold into slavery.[76] - -Egypt - -Further information: Siege of Gaza (332 BCE) -When Alexander destroyed Tyre, most of the towns on the route to Egypt quickly capitulated. However, Alexander was met with resistance at Gaza. The stronghold was heavily fortified and built on a hill, requiring a siege. When "his engineers pointed out to him that because of the height of the mound it would be impossible... this encouraged Alexander all the more to make the attempt".[77] After three unsuccessful assaults, the stronghold fell, but not before Alexander had received a serious shoulder wound. As in Tyre, men of military age were put to the sword and the women and children were sold into slavery.[78] -""" diff --git a/tests/local_testing/openai_batch_completions.jsonl b/tests/local_testing/openai_batch_completions.jsonl deleted file mode 100644 index 05448952a0f..00000000000 --- a/tests/local_testing/openai_batch_completions.jsonl +++ /dev/null @@ -1,2 +0,0 @@ -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} \ No newline at end of file diff --git a/tests/local_testing/openai_batch_completions_router.jsonl b/tests/local_testing/openai_batch_completions_router.jsonl deleted file mode 100644 index 8a4c99ca85e..00000000000 --- a/tests/local_testing/openai_batch_completions_router.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"custom_id": "task-0", "method": "POST", "url": "/chat/completions", "body": {"model": "my-custom-name", "messages": [{"role": "system", "content": "You are an AI assistant that helps people find information."}, {"role": "user", "content": "When was Microsoft founded?"}]}} -{"custom_id": "task-1", "method": "POST", "url": "/chat/completions", "body": {"model": "my-custom-name", "messages": [{"role": "system", "content": "You are an AI assistant that helps people find information."}, {"role": "user", "content": "When was the first XBOX released?"}]}} -{"custom_id": "task-2", "method": "POST", "url": "/chat/completions", "body": {"model": "my-custom-name", "messages": [{"role": "system", "content": "You are an AI assistant that helps people find information."}, {"role": "user", "content": "What is Altair Basic?"}]}} \ No newline at end of file diff --git a/tests/local_testing/speech_vertex.mp3 b/tests/local_testing/speech_vertex.mp3 deleted file mode 100644 index c676110335e..00000000000 Binary files a/tests/local_testing/speech_vertex.mp3 and /dev/null differ diff --git a/tests/local_testing/test_configs/custom_auth.py b/tests/local_testing/test_configs/custom_auth.py deleted file mode 100644 index 1b6bec43b5e..00000000000 --- a/tests/local_testing/test_configs/custom_auth.py +++ /dev/null @@ -1,22 +0,0 @@ -from litellm.proxy._types import UserAPIKeyAuth -from fastapi import Request -from dotenv import load_dotenv -import os - -load_dotenv() - - -async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth: - try: - print(f"api_key: {api_key}") - if api_key == "": - raise Exception( - f"CustomAuth - Malformed API Key passed in. Ensure Key has `Bearer` prefix" - ) - if api_key == f"{os.getenv('PROXY_MASTER_KEY')}-1234": - return UserAPIKeyAuth(api_key=api_key) - raise Exception - except Exception as e: - if len(str(e)) > 0: - raise e - raise Exception("Failed custom auth") diff --git a/tests/local_testing/test_configs/custom_callbacks.py b/tests/local_testing/test_configs/custom_callbacks.py deleted file mode 100644 index 42f88b5d19b..00000000000 --- a/tests/local_testing/test_configs/custom_callbacks.py +++ /dev/null @@ -1,121 +0,0 @@ -from litellm.integrations.custom_logger import CustomLogger -import inspect -import litellm - - -class testCustomCallbackProxy(CustomLogger): - def __init__(self): - self.success: bool = False # type: ignore - self.failure: bool = False # type: ignore - self.async_success: bool = False # type: ignore - self.async_success_embedding: bool = False # type: ignore - self.async_failure: bool = False # type: ignore - self.async_failure_embedding: bool = False # type: ignore - - self.async_completion_kwargs = None # type: ignore - self.async_embedding_kwargs = None # type: ignore - self.async_embedding_response = None # type: ignore - - self.async_completion_kwargs_fail = None # type: ignore - self.async_embedding_kwargs_fail = None # type: ignore - - self.streaming_response_obj = None # type: ignore - blue_color_code = "\033[94m" - reset_color_code = "\033[0m" - print(f"{blue_color_code}Initialized LiteLLM custom logger") - try: - print(f"Logger Initialized with following methods:") - methods = [ - method - for method in dir(self) - if inspect.ismethod(getattr(self, method)) - ] - - # Pretty print the methods - for method in methods: - print(f" - {method}") - print(f"{reset_color_code}") - except Exception: - pass - - def log_pre_api_call(self, model, messages, kwargs): - print(f"Pre-API Call") - - def log_post_api_call(self, kwargs, response_obj, start_time, end_time): - print(f"Post-API Call") - - def log_stream_event(self, kwargs, response_obj, start_time, end_time): - print(f"On Stream") - - def log_success_event(self, kwargs, response_obj, start_time, end_time): - print(f"On Success") - self.success = True - - def log_failure_event(self, kwargs, response_obj, start_time, end_time): - print(f"On Failure") - self.failure = True - - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - print(f"On Async success") - self.async_success = True - print("Value of async success: ", self.async_success) - print("\n kwargs: ", kwargs) - if ( - kwargs.get("model") == "azure-embedding-model" - or kwargs.get("model") == "ada" - ): - print("Got an embedding model", kwargs.get("model")) - print("Setting embedding success to True") - self.async_success_embedding = True - print("Value of async success embedding: ", self.async_success_embedding) - self.async_embedding_kwargs = kwargs - self.async_embedding_response = response_obj - if kwargs.get("stream") == True: - self.streaming_response_obj = response_obj - - self.async_completion_kwargs = kwargs - - model = kwargs.get("model", None) - messages = kwargs.get("messages", None) - user = kwargs.get("user", None) - - # Access litellm_params passed to litellm.completion(), example access `metadata` - litellm_params = kwargs.get("litellm_params", {}) - metadata = litellm_params.get( - "metadata", {} - ) # headers passed to LiteLLM proxy, can be found here - - # Calculate cost using litellm.completion_cost() - cost = litellm.completion_cost(completion_response=response_obj) - response = response_obj - # tokens used in response - usage = response_obj["usage"] - - print("\n\n in custom callback vars my custom logger, ", vars(my_custom_logger)) - - print( - f""" - Model: {model}, - Messages: {messages}, - User: {user}, - Usage: {usage}, - Cost: {cost}, - Response: {response} - Proxy Metadata: {metadata} - """ - ) - return - - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): - print(f"On Async Failure") - self.async_failure = True - print("Value of async failure: ", self.async_failure) - print("\n kwargs: ", kwargs) - if kwargs.get("model") == "text-embedding-ada-002": - self.async_failure_embedding = True - self.async_embedding_kwargs_fail = kwargs - - self.async_completion_kwargs_fail = kwargs - - -my_custom_logger = testCustomCallbackProxy() diff --git a/tests/local_testing/test_configs/test_bad_config.yaml b/tests/local_testing/test_configs/test_bad_config.yaml deleted file mode 100644 index 4bc4c7cc541..00000000000 --- a/tests/local_testing/test_configs/test_bad_config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -model_list: - - model_name: gpt-3.5-turbo - litellm_params: - api_key: bad-key - model: gpt-3.5-turbo - - model_name: working-azure-gpt-3.5-turbo - litellm_params: - model: azure/gpt-4.1-mini - api_base: os.environ/AZURE_AI_API_BASE - api_key: os.environ/AZURE_AI_API_KEY - - model_name: azure-gpt-3.5-turbo - litellm_params: - model: azure/gpt-4.1-mini - api_base: os.environ/AZURE_AI_API_BASE - api_key: bad-key - - model_name: azure-embedding - litellm_params: - model: azure/text-embedding-ada-002 - api_base: os.environ/AZURE_AI_API_BASE - api_key: bad-key - \ No newline at end of file diff --git a/tests/local_testing/test_configs/test_cloudflare_azure_with_cache_config.yaml b/tests/local_testing/test_configs/test_cloudflare_azure_with_cache_config.yaml deleted file mode 100644 index 24240008fe2..00000000000 --- a/tests/local_testing/test_configs/test_cloudflare_azure_with_cache_config.yaml +++ /dev/null @@ -1,17 +0,0 @@ -model_list: - - model_name: azure-cloudflare - litellm_params: - model: azure/gpt-4.1-mini - api_base: https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/openai-gpt-4-test-v-1 - api_key: os.environ/AZURE_AI_API_KEY - api_version: 2023-07-01-preview - -litellm_settings: - set_verbose: True - cache: True # set cache responses to True - cache_params: # set cache params for s3 - type: s3 - s3_bucket_name: litellm-my-test-bucket-2 # AWS Bucket Name for S3 - s3_region_name: us-east-1 # AWS Region Name for S3 - s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # AWS Access Key ID for S3 - s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3 \ No newline at end of file diff --git a/tests/local_testing/test_configs/test_config.yaml b/tests/local_testing/test_configs/test_config.yaml deleted file mode 100644 index a711b65ea03..00000000000 --- a/tests/local_testing/test_configs/test_config.yaml +++ /dev/null @@ -1,28 +0,0 @@ -general_settings: - database_url: os.environ/DATABASE_URL - master_key: os.environ/PROXY_MASTER_KEY -litellm_settings: - drop_params: true - success_callback: ["langfuse"] - -model_list: -- litellm_params: - api_base: https://my-endpoint-europe-berri-992.openai.azure.com/ - api_key: os.environ/AZURE_EUROPE_API_KEY - model: azure/gpt-35-turbo - model_name: azure-model -- litellm_params: - api_base: https://my-endpoint-canada-berri992.openai.azure.com - api_key: os.environ/AZURE_CANADA_API_KEY - model: azure/gpt-35-turbo - model_name: azure-model -- litellm_params: - api_base: https://openai-france-1234.openai.azure.com - api_key: os.environ/AZURE_FRANCE_API_KEY - model: azure/gpt-turbo - model_name: azure-model -- litellm_params: - model: gpt-3.5-turbo - model_info: - description: this is a test openai model - model_name: test_openai_models diff --git a/tests/local_testing/test_configs/test_config_custom_auth.yaml b/tests/local_testing/test_configs/test_config_custom_auth.yaml deleted file mode 100644 index 33088bd1ce1..00000000000 --- a/tests/local_testing/test_configs/test_config_custom_auth.yaml +++ /dev/null @@ -1,11 +0,0 @@ -model_list: - - model_name: "openai-model" - litellm_params: - model: "gpt-3.5-turbo" - -litellm_settings: - drop_params: True - set_verbose: True - -general_settings: - custom_auth: custom_auth.user_api_key_auth \ No newline at end of file diff --git a/tests/local_testing/test_configs/test_config_no_auth.yaml b/tests/local_testing/test_configs/test_config_no_auth.yaml deleted file mode 100644 index f4896217049..00000000000 --- a/tests/local_testing/test_configs/test_config_no_auth.yaml +++ /dev/null @@ -1,127 +0,0 @@ -model_list: -- litellm_params: - api_base: https://my-endpoint-europe-berri-992.openai.azure.com/ - api_key: os.environ/AZURE_EUROPE_API_KEY - model: azure/gpt-35-turbo - model_name: azure-model -- litellm_params: - api_base: https://my-endpoint-canada-berri992.openai.azure.com - api_key: os.environ/AZURE_CANADA_API_KEY - model: azure/gpt-35-turbo - model_name: azure-model -- litellm_params: - api_base: https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/openai-gpt-4-test-v-1 - api_key: os.environ/AZURE_AI_API_KEY - model: azure/gpt-4.1-mini - model_name: azure-cloudflare-model -- litellm_params: - api_base: https://openai-france-1234.openai.azure.com - api_key: os.environ/AZURE_FRANCE_API_KEY - model: azure/gpt-turbo - model_name: azure-model -- litellm_params: - model: gpt-3.5-turbo - model_info: - description: this is a test openai model - model_name: test_openai_models -- litellm_params: - model: gpt-3.5-turbo - model_info: - description: this is a test openai model - id: 56f1bd94-3b54-4b67-9ea2-7c70e9a3a709 - model_name: test_openai_models -- litellm_params: - model: gpt-3.5-turbo - model_info: - description: this is a test openai model - id: 4d1ee26c-abca-450c-8744-8e87fd6755e9 - model_name: test_openai_models -- litellm_params: - model: gpt-3.5-turbo - model_info: - description: this is a test openai model - id: 00e19c0f-b63d-42bb-88e9-016fb0c60764 - model_name: test_openai_models -- litellm_params: - model: gpt-3.5-turbo - model_info: - description: this is a test openai model - id: 79fc75bf-8e1b-47d5-8d24-9365a854af03 - model_name: test_openai_models -- litellm_params: - api_base: os.environ/AZURE_AI_API_BASE - api_key: os.environ/AZURE_AI_API_KEY - api_version: 2023-07-01-preview - model: azure/text-embedding-ada-002 - model_info: - mode: embedding - model_name: azure-embedding-model -- litellm_params: - model: gpt-3.5-turbo - model_info: - description: this is a test openai model - id: 55848c55-4162-40f9-a6e2-9a722b9ef404 - model_name: test_openai_models -- litellm_params: - model: gpt-3.5-turbo - model_info: - description: this is a test openai model - id: 34339b1e-e030-4bcc-a531-c48559f10ce4 - model_name: test_openai_models -- litellm_params: - model: gpt-3.5-turbo - model_info: - description: this is a test openai model - id: f6f74e14-ac64-4403-9365-319e584dcdc5 - model_name: test_openai_models -- litellm_params: - model: gpt-3.5-turbo - model_info: - description: this is a test openai model - id: 9b1ef341-322c-410a-8992-903987fef439 - model_name: test_openai_models -- litellm_params: - model: dall-e-3 - model_info: - mode: image_generation - model_name: dall-e-3 -- litellm_params: - api_base: os.environ/AZURE_SWEDEN_API_BASE - api_key: os.environ/AZURE_SWEDEN_API_KEY - api_version: 2023-12-01-preview - model: azure/dall-e-3-test - model_info: - mode: image_generation - model_name: dall-e-3 -- litellm_params: - api_base: os.environ/AZURE_AI_API_BASE - api_key: os.environ/AZURE_AI_API_KEY - api_version: 2023-06-01-preview - model: azure/ - model_info: - mode: image_generation - model_name: dall-e-2 -- litellm_params: - api_base: os.environ/AZURE_AI_API_BASE - api_key: os.environ/AZURE_AI_API_KEY - api_version: 2023-07-01-preview - model: azure/text-embedding-ada-002 - model_info: - base_model: text-embedding-ada-002 - mode: embedding - model_name: text-embedding-ada-002 -- litellm_params: - model: gpt-3.5-turbo - model_info: - description: this is a test openai model - id: 34cb2419-7c63-44ae-a189-53f1d1ce5953 - model_name: test_openai_models -- litellm_params: - model: amazon.titan-embed-text-v1 - model_name: amazon-embeddings -- litellm_params: - model: gpt-3.5-turbo - model_info: - description: this is a test openai model - id: 753dca9a-898d-4ff7-9961-5acf7cdf38cf - model_name: test_openai_models diff --git a/tests/local_testing/test_configs/test_custom_logger.yaml b/tests/local_testing/test_configs/test_custom_logger.yaml deleted file mode 100644 index 464d66bd783..00000000000 --- a/tests/local_testing/test_configs/test_custom_logger.yaml +++ /dev/null @@ -1,26 +0,0 @@ -model_list: - - model_name: Azure OpenAI GPT-4 Canada - litellm_params: - model: azure/gpt-4.1-mini - api_base: os.environ/AZURE_AI_API_BASE - api_key: os.environ/AZURE_AI_API_KEY - api_version: "2023-07-01-preview" - model_info: - mode: chat - input_cost_per_token: 0.0002 - id: gm - - model_name: azure-embedding-model - litellm_params: - model: azure/text-embedding-ada-002 - api_base: os.environ/AZURE_AI_API_BASE - api_key: os.environ/AZURE_AI_API_KEY - api_version: "2023-07-01-preview" - model_info: - mode: embedding - input_cost_per_token: 0.002 - id: hello - -litellm_settings: - drop_params: True - set_verbose: True - callbacks: custom_callbacks.my_custom_logger \ No newline at end of file diff --git a/tests/local_testing/test_configs/test_guardrails_config.yaml b/tests/local_testing/test_configs/test_guardrails_config.yaml deleted file mode 100644 index f09ff9d1bc4..00000000000 --- a/tests/local_testing/test_configs/test_guardrails_config.yaml +++ /dev/null @@ -1,32 +0,0 @@ - - -model_list: -- litellm_params: - api_base: https://my-endpoint-europe-berri-992.openai.azure.com/ - api_key: os.environ/AZURE_EUROPE_API_KEY - model: azure/gpt-35-turbo - model_name: azure-model -- litellm_params: - api_base: https://my-endpoint-canada-berri992.openai.azure.com - api_key: os.environ/AZURE_CANADA_API_KEY - model: azure/gpt-35-turbo - model_name: azure-model -- litellm_params: - api_base: https://openai-france-1234.openai.azure.com - api_key: os.environ/AZURE_FRANCE_API_KEY - model: azure/gpt-turbo - model_name: azure-model - - - -litellm_settings: - guardrails: - - prompt_injection: - callbacks: [lakera_prompt_injection, detect_prompt_injection] - default_on: true - - hide_secrets: - callbacks: [hide_secrets] - default_on: true - - moderations: - callbacks: [openai_moderations] - default_on: false \ No newline at end of file diff --git a/tests/local_testing/test_disk_cache_unit_tests.py b/tests/local_testing/test_disk_cache_unit_tests.py deleted file mode 100644 index c777d04ecce..00000000000 --- a/tests/local_testing/test_disk_cache_unit_tests.py +++ /dev/null @@ -1,11 +0,0 @@ -from cache_unit_tests import LLMCachingUnitTests -from litellm.caching import LiteLLMCacheType - - -class TestDiskCacheUnitTests(LLMCachingUnitTests): - def get_cache_type(self) -> LiteLLMCacheType: - return LiteLLMCacheType.DISK - - -# if __name__ == "__main__": -# pytest.main([__file__, "-v", "-s"]) diff --git a/tests/local_testing/test_gcs_cache_unit_tests.py b/tests/local_testing/test_gcs_cache_unit_tests.py deleted file mode 100644 index 4604dc0d0ae..00000000000 --- a/tests/local_testing/test_gcs_cache_unit_tests.py +++ /dev/null @@ -1,7 +0,0 @@ -from cache_unit_tests import LLMCachingUnitTests -from litellm.caching import LiteLLMCacheType - - -class TestGCSCacheUnitTests(LLMCachingUnitTests): - def get_cache_type(self) -> LiteLLMCacheType: - return LiteLLMCacheType.GCS diff --git a/tests/local_testing/user_cost.json b/tests/local_testing/user_cost.json deleted file mode 100644 index bb15ef812f5..00000000000 --- a/tests/local_testing/user_cost.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "1234": { - "total_budget": 10, - "current_cost": 7.3e-05, - "model_cost": { - "gpt-3.5-turbo": 7.3e-05 - } - }, - "12345": { - "total_budget": 0 - } -} \ No newline at end of file diff --git a/tests/local_testing/vertex_ai.jsonl b/tests/local_testing/vertex_ai.jsonl deleted file mode 100644 index 6ec5baaf96b..00000000000 --- a/tests/local_testing/vertex_ai.jsonl +++ /dev/null @@ -1,22 +0,0 @@ -[ - { - "messages": [ - { - "role": "system", - "content": "You should classify the text into one of the following classes:[business, entertainment]" - }, - { "role": "user", "content": "Diversify your investment portfolio" }, - { "role": "model", "content": "business" } - ] - }, - { - "messages": [ - { - "role": "system", - "content": "You should classify the text into one of the following classes:[business, entertainment]" - }, - { "role": "user", "content": "Watch a live concert" }, - { "role": "model", "content": "entertainment" } - ] - } -] \ No newline at end of file diff --git a/tests/local_testing/vertex_batch_completions.jsonl b/tests/local_testing/vertex_batch_completions.jsonl deleted file mode 100644 index ec899f8fc43..00000000000 --- a/tests/local_testing/vertex_batch_completions.jsonl +++ /dev/null @@ -1,2 +0,0 @@ -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-1.5-flash-001", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-1.5-flash-001", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} \ No newline at end of file diff --git a/tests/local_testing/vertex_key.json b/tests/local_testing/vertex_key.json deleted file mode 100644 index 800969fb305..00000000000 --- a/tests/local_testing/vertex_key.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "type": "service_account", - "project_id": "litellm-ci-cd", - "private_key_id": "", - "private_key": "", - "client_email": "test-litellm-ci-cd@litellm-ci-cd.iam.gserviceaccount.com", - "client_id": "116563532503305622785", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-litellm-ci-cd%40litellm-ci-cd.iam.gserviceaccount.com", - "universe_domain": "googleapis.com" -} diff --git a/tests/local_testing/whitelisted_bedrock_models.txt b/tests/local_testing/whitelisted_bedrock_models.txt deleted file mode 100644 index 9b66569ff9e..00000000000 --- a/tests/local_testing/whitelisted_bedrock_models.txt +++ /dev/null @@ -1,142 +0,0 @@ -ai21.j2-mid-v1 -ai21.j2-ultra-v1 -ai21.jamba-instruct-v1:0 -amazon.titan-text-lite-v1 -amazon.titan-text-express-v1 -amazon.titan-text-premier-v1:0 -mistral.mistral-7b-instruct-v0:2 -mistral.mixtral-8x7b-instruct-v0:1 -mistral.mistral-large-2402-v1:0 -mistral.mistral-large-2407-v1:0 -mistral.mistral-small-2402-v1:0 -bedrock/us-west-2/mistral.mixtral-8x7b-instruct-v0:1 -bedrock/us-east-1/mistral.mixtral-8x7b-instruct-v0:1 -bedrock/eu-west-3/mistral.mixtral-8x7b-instruct-v0:1 -bedrock/us-west-2/mistral.mistral-7b-instruct-v0:2 -bedrock/us-east-1/mistral.mistral-7b-instruct-v0:2 -bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2 -bedrock/us-east-1/mistral.mistral-large-2402-v1:0 -bedrock/us-west-2/mistral.mistral-large-2402-v1:0 -bedrock/eu-west-3/mistral.mistral-large-2402-v1:0 -anthropic.claude-3-sonnet-20240229-v1:0 -anthropic.claude-haiku-4-5-20251001-v1:0 -anthropic.claude-3-7-sonnet-20250219-v1:0 -anthropic.claude-haiku-4-5-20251001-v1:0 -anthropic.claude-3-haiku-20240307-v1:0 -anthropic.claude-3-5-haiku-20241022-v1:0 -anthropic.claude-3-7-sonnet-20250219-v1:0 -us.anthropic.claude-3-sonnet-20240229-v1:0 -us.anthropic.claude-haiku-4-5-20251001-v1:0 -us.anthropic.claude-3-7-sonnet-20250219-v1:0 -us.anthropic.claude-haiku-4-5-20251001-v1:0 -us.anthropic.claude-3-haiku-20240307-v1:0 -us.anthropic.claude-3-5-haiku-20241022-v1:0 -us.anthropic.claude-3-7-sonnet-20250219-v1:0 -eu.anthropic.claude-3-sonnet-20240229-v1:0 -eu.anthropic.claude-haiku-4-5-20251001-v1:0 -eu.anthropic.claude-haiku-4-5-20251001-v1:0 -eu.anthropic.claude-3-haiku-20240307-v1:0 -eu.anthropic.claude-3-5-haiku-20241022-v1:0 -eu.anthropic.claude-3-7-sonnet-20250219-v1:0 -anthropic.claude-v1 -bedrock/us-east-1/anthropic.claude-v1 -bedrock/us-west-2/anthropic.claude-v1 -bedrock/ap-northeast-1/anthropic.claude-v1 -bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-v1 -bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-v1 -bedrock/eu-central-1/anthropic.claude-v1 -bedrock/eu-central-1/1-month-commitment/anthropic.claude-v1 -bedrock/eu-central-1/6-month-commitment/anthropic.claude-v1 -bedrock/us-east-1/1-month-commitment/anthropic.claude-v1 -bedrock/us-east-1/6-month-commitment/anthropic.claude-v1 -bedrock/us-west-2/1-month-commitment/anthropic.claude-v1 -bedrock/us-west-2/6-month-commitment/anthropic.claude-v1 -anthropic.claude-v2 -bedrock/us-east-1/anthropic.claude-v2 -bedrock/us-west-2/anthropic.claude-v2 -bedrock/ap-northeast-1/anthropic.claude-v2 -bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-v2 -bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-v2 -bedrock/eu-central-1/anthropic.claude-v2 -bedrock/eu-central-1/1-month-commitment/anthropic.claude-v2 -bedrock/eu-central-1/6-month-commitment/anthropic.claude-v2 -bedrock/us-east-1/1-month-commitment/anthropic.claude-v2 -bedrock/us-east-1/6-month-commitment/anthropic.claude-v2 -bedrock/us-west-2/1-month-commitment/anthropic.claude-v2 -bedrock/us-west-2/6-month-commitment/anthropic.claude-v2 -anthropic.claude-v2:1 -bedrock/us-east-1/anthropic.claude-v2:1 -bedrock/us-west-2/anthropic.claude-v2:1 -bedrock/ap-northeast-1/anthropic.claude-v2:1 -bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-v2:1 -bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-v2:1 -bedrock/eu-central-1/anthropic.claude-v2:1 -bedrock/eu-central-1/1-month-commitment/anthropic.claude-v2:1 -bedrock/eu-central-1/6-month-commitment/anthropic.claude-v2:1 -bedrock/us-east-1/1-month-commitment/anthropic.claude-v2:1 -bedrock/us-east-1/6-month-commitment/anthropic.claude-v2:1 -bedrock/us-west-2/1-month-commitment/anthropic.claude-v2:1 -bedrock/us-west-2/6-month-commitment/anthropic.claude-v2:1 -anthropic.claude-instant-v1 -bedrock/us-east-1/anthropic.claude-instant-v1 -bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1 -bedrock/us-east-1/6-month-commitment/anthropic.claude-instant-v1 -bedrock/us-west-2/1-month-commitment/anthropic.claude-instant-v1 -bedrock/us-west-2/6-month-commitment/anthropic.claude-instant-v1 -bedrock/us-west-2/anthropic.claude-instant-v1 -bedrock/ap-northeast-1/anthropic.claude-instant-v1 -bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-instant-v1 -bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-instant-v1 -bedrock/eu-central-1/anthropic.claude-instant-v1 -bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1 -bedrock/eu-central-1/6-month-commitment/anthropic.claude-instant-v1 -cohere.command-text-v14 -bedrock/*/1-month-commitment/cohere.command-text-v14 -bedrock/*/6-month-commitment/cohere.command-text-v14 -cohere.command-light-text-v14 -bedrock/*/1-month-commitment/cohere.command-light-text-v14 -bedrock/*/6-month-commitment/cohere.command-light-text-v14 -cohere.command-r-plus-v1:0 -cohere.command-r-v1:0 -meta.llama3-3-70b-instruct-v1:0 -meta.llama2-13b-chat-v1 -meta.llama2-70b-chat-v1 -meta.llama3-8b-instruct-v1:0 -bedrock/us-east-1/meta.llama3-8b-instruct-v1:0 -bedrock/us-west-1/meta.llama3-8b-instruct-v1:0 -bedrock/ap-south-1/meta.llama3-8b-instruct-v1:0 -bedrock/ca-central-1/meta.llama3-8b-instruct-v1:0 -bedrock/eu-west-1/meta.llama3-8b-instruct-v1:0 -bedrock/eu-west-2/meta.llama3-8b-instruct-v1:0 -bedrock/sa-east-1/meta.llama3-8b-instruct-v1:0 -meta.llama3-70b-instruct-v1:0 -bedrock/us-east-1/meta.llama3-70b-instruct-v1:0 -bedrock/us-west-1/meta.llama3-70b-instruct-v1:0 -bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0 -bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0 -bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0 -bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0 -bedrock/sa-east-1/meta.llama3-70b-instruct-v1:0 -meta.llama3-1-8b-instruct-v1:0 -us.meta.llama3-1-8b-instruct-v1:0 -meta.llama3-1-70b-instruct-v1:0 -us.meta.llama3-1-70b-instruct-v1:0 -meta.llama3-1-405b-instruct-v1:0 -us.meta.llama3-1-405b-instruct-v1:0 -meta.llama3-2-1b-instruct-v1:0 -us.meta.llama3-2-1b-instruct-v1:0 -eu.meta.llama3-2-1b-instruct-v1:0 -meta.llama3-2-3b-instruct-v1:0 -us.meta.llama3-2-3b-instruct-v1:0 -eu.meta.llama3-2-3b-instruct-v1:0 -meta.llama3-2-11b-instruct-v1:0 -us.meta.llama3-2-11b-instruct-v1:0 -meta.llama3-2-90b-instruct-v1:0 -us.meta.llama3-2-90b-instruct-v1:0 -bedrock/ap-northeast-1/qwen.qwen3-next-80b-a3b -bedrock/ap-south-1/qwen.qwen3-next-80b-a3b -bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b -bedrock/eu-west-1/qwen.qwen3-next-80b-a3b -bedrock/eu-west-2/qwen.qwen3-next-80b-a3b -bedrock/sa-east-1/qwen.qwen3-next-80b-a3b -bedrock/eu-west-2/nvidia.nemotron-super-3-120b diff --git a/tests/pass_through_tests/package-lock.json b/tests/pass_through_tests/package-lock.json deleted file mode 100644 index 2f8e7fe21b2..00000000000 --- a/tests/pass_through_tests/package-lock.json +++ /dev/null @@ -1,4117 +0,0 @@ -{ - "name": "litellm-pass-through-tests", - "version": "0.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "litellm-pass-through-tests", - "version": "0.0.0", - "dependencies": { - "@google-cloud/vertexai": "1.12.0", - "@google/generative-ai": "0.24.1" - }, - "devDependencies": { - "jest": "29.7.0" - } - }, - "node_modules/@babel/code-frame": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", - "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.28.5", - "js-tokens": "^4.0.0", - "picocolors": "^1.1.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/compat-data": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.0.tgz", - "integrity": "sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/core": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz", - "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.29.0", - "@babel/generator": "^7.29.0", - "@babel/helper-compilation-targets": "^7.28.6", - "@babel/helper-module-transforms": "^7.28.6", - "@babel/helpers": "^7.28.6", - "@babel/parser": "^7.29.0", - "@babel/template": "^7.28.6", - "@babel/traverse": "^7.29.0", - "@babel/types": "^7.29.0", - "@jridgewell/remapping": "^2.3.5", - "convert-source-map": "^2.0.0", - "debug": "^4.1.0", - "gensync": "^1.0.0-beta.2", - "json5": "^2.2.3", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/babel" - } - }, - "node_modules/@babel/generator": { - "version": "7.29.1", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.1.tgz", - "integrity": "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.29.0", - "@babel/types": "^7.29.0", - "@jridgewell/gen-mapping": "^0.3.12", - "@jridgewell/trace-mapping": "^0.3.28", - "jsesc": "^3.0.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-compilation-targets": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.28.6.tgz", - "integrity": "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/compat-data": "^7.28.6", - "@babel/helper-validator-option": "^7.27.1", - "browserslist": "^4.24.0", - "lru-cache": "^5.1.1", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-globals": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", - "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-imports": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.28.6.tgz", - "integrity": "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/traverse": "^7.28.6", - "@babel/types": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-transforms": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.6.tgz", - "integrity": "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-module-imports": "^7.28.6", - "@babel/helper-validator-identifier": "^7.28.5", - "@babel/traverse": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/@babel/helper-plugin-utils": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.28.6.tgz", - "integrity": "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-string-parser": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", - "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", - "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-option": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", - "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helpers": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.2.tgz", - "integrity": "sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.28.6", - "@babel/types": "^7.29.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/parser": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.2.tgz", - "integrity": "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.29.0" - }, - "bin": { - "parser": "bin/babel-parser.js" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/plugin-syntax-async-generators": { - "version": "7.8.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", - "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-bigint": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", - "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-class-properties": { - "version": "7.12.13", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", - "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.12.13" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-class-static-block": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", - "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-import-attributes": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.28.6.tgz", - "integrity": "sha512-jiLC0ma9XkQT3TKJ9uYvlakm66Pamywo+qwL+oL8HJOvc6TWdZXVfhqJr8CCzbSGUAbDOzlGHJC1U+vRfLQDvw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-import-meta": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", - "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-json-strings": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", - "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-jsx": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.28.6.tgz", - "integrity": "sha512-wgEmr06G6sIpqr8YDwA2dSRTE3bJ+V0IfpzfSY3Lfgd7YWOaAdlykvJi13ZKBt8cZHfgH1IXN+CL656W3uUa4w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-logical-assignment-operators": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", - "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", - "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-numeric-separator": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", - "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-object-rest-spread": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", - "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-optional-catch-binding": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", - "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-optional-chaining": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", - "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-private-property-in-object": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", - "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-top-level-await": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", - "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-typescript": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.28.6.tgz", - "integrity": "sha512-+nDNmQye7nlnuuHDboPbGm00Vqg3oO8niRRL27/4LYHUsHYh0zJ1xWOz0uRwNFmM1Avzk8wZbc6rdiYhomzv/A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/template": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", - "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.28.6", - "@babel/parser": "^7.28.6", - "@babel/types": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.0.tgz", - "integrity": "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.29.0", - "@babel/generator": "^7.29.0", - "@babel/helper-globals": "^7.28.0", - "@babel/parser": "^7.29.0", - "@babel/template": "^7.28.6", - "@babel/types": "^7.29.0", - "debug": "^4.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/types": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", - "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.28.5" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@bcoe/v8-coverage": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", - "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@google-cloud/vertexai": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/@google-cloud/vertexai/-/vertexai-1.12.0.tgz", - "integrity": "sha512-XMJIk7GIeavFLP5A3YEUlowKa5Y5PZRrnnuTJcqR0k+lFKkv7+IWpdRp+Xbqb8xNDrvQaE2hP2RYPUylyD5EdA==", - "license": "Apache-2.0", - "dependencies": { - "@google/genai": "^1.45.0", - "google-auth-library": "^9.1.0" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@google/genai": { - "version": "1.51.0", - "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.51.0.tgz", - "integrity": "sha512-vTZZF3CSimN7cn2zsLpW2p5WF0eZa5Gz69ITMPCNHpPrDlAstOfGifSfi0p/s9Z9400f7xJRkgvkQNrcM7pJ6w==", - "hasInstallScript": true, - "license": "Apache-2.0", - "dependencies": { - "google-auth-library": "^10.3.0", - "p-retry": "^4.6.2", - "protobufjs": "^7.5.4", - "ws": "^8.18.0" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@modelcontextprotocol/sdk": "^1.25.2" - }, - "peerDependenciesMeta": { - "@modelcontextprotocol/sdk": { - "optional": true - } - } - }, - "node_modules/@google/generative-ai": { - "version": "0.24.1", - "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.24.1.tgz", - "integrity": "sha512-MqO+MLfM6kjxcKoy0p1wRzG3b4ZZXtPI+z2IE26UogS2Cm/XHO+7gGRBh6gcJsOiIVoH93UwKvW4HdgiOZCy9Q==", - "license": "Apache-2.0", - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@istanbuljs/load-nyc-config": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", - "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "camelcase": "^5.3.1", - "find-up": "^4.1.0", - "get-package-type": "^0.1.0", - "js-yaml": "^3.13.1", - "resolve-from": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/schema": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.6.tgz", - "integrity": "sha512-+Sg6GCR/wy1oSmQDFq4LQDAhm3ETKnorxN+y5nbLULOR3P0c14f2Wurzj3/xqPXtasLFfHd5iRFQ7AJt4KH2cw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jest/console": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz", - "integrity": "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/core": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz", - "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/console": "^29.7.0", - "@jest/reporters": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "ansi-escapes": "^4.2.1", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "exit": "^0.1.2", - "graceful-fs": "^4.2.9", - "jest-changed-files": "^29.7.0", - "jest-config": "^29.7.0", - "jest-haste-map": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-regex-util": "^29.6.3", - "jest-resolve": "^29.7.0", - "jest-resolve-dependencies": "^29.7.0", - "jest-runner": "^29.7.0", - "jest-runtime": "^29.7.0", - "jest-snapshot": "^29.7.0", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "jest-watcher": "^29.7.0", - "micromatch": "^4.0.4", - "pretty-format": "^29.7.0", - "slash": "^3.0.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/@jest/environment": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz", - "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/fake-timers": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "jest-mock": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/expect": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz", - "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "expect": "^29.7.0", - "jest-snapshot": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/expect-utils": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz", - "integrity": "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==", - "dev": true, - "license": "MIT", - "dependencies": { - "jest-get-type": "^29.6.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/fake-timers": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz", - "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@sinonjs/fake-timers": "^10.0.2", - "@types/node": "*", - "jest-message-util": "^29.7.0", - "jest-mock": "^29.7.0", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/globals": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz", - "integrity": "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/expect": "^29.7.0", - "@jest/types": "^29.6.3", - "jest-mock": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/reporters": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz", - "integrity": "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@bcoe/v8-coverage": "^0.2.3", - "@jest/console": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@jridgewell/trace-mapping": "^0.3.18", - "@types/node": "*", - "chalk": "^4.0.0", - "collect-v8-coverage": "^1.0.0", - "exit": "^0.1.2", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", - "istanbul-lib-coverage": "^3.0.0", - "istanbul-lib-instrument": "^6.0.0", - "istanbul-lib-report": "^3.0.0", - "istanbul-lib-source-maps": "^4.0.0", - "istanbul-reports": "^3.1.3", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0", - "jest-worker": "^29.7.0", - "slash": "^3.0.0", - "string-length": "^4.0.1", - "strip-ansi": "^6.0.0", - "v8-to-istanbul": "^9.0.1" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/@jest/schemas": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", - "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.27.8" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/source-map": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz", - "integrity": "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.18", - "callsites": "^3.0.0", - "graceful-fs": "^4.2.9" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/test-result": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz", - "integrity": "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/console": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/istanbul-lib-coverage": "^2.0.0", - "collect-v8-coverage": "^1.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/test-sequencer": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz", - "integrity": "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/test-result": "^29.7.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/transform": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz", - "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.11.6", - "@jest/types": "^29.6.3", - "@jridgewell/trace-mapping": "^0.3.18", - "babel-plugin-istanbul": "^6.1.1", - "chalk": "^4.0.0", - "convert-source-map": "^2.0.0", - "fast-json-stable-stringify": "^2.1.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "jest-regex-util": "^29.6.3", - "jest-util": "^29.7.0", - "micromatch": "^4.0.4", - "pirates": "^4.0.4", - "slash": "^3.0.0", - "write-file-atomic": "^4.0.2" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/types": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", - "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.13", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", - "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/remapping": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", - "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.31", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", - "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" - } - }, - "node_modules/@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/codegen": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", - "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/fetch": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", - "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.1" - } - }, - "node_modules/@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.2.tgz", - "integrity": "sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", - "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", - "license": "BSD-3-Clause" - }, - "node_modules/@sinclair/typebox": { - "version": "0.27.10", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.10.tgz", - "integrity": "sha512-MTBk/3jGLNB2tVxv6uLlFh1iu64iYOQ2PbdOSK3NW8JZsmlaOh2q6sdtKowBhfw8QFLmYNzTW4/oK4uATIi6ZA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@sinonjs/commons": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", - "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "type-detect": "4.0.8" - } - }, - "node_modules/@sinonjs/fake-timers": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", - "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@sinonjs/commons": "^3.0.0" - } - }, - "node_modules/@types/babel__core": { - "version": "7.20.5", - "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", - "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.20.7", - "@babel/types": "^7.20.7", - "@types/babel__generator": "*", - "@types/babel__template": "*", - "@types/babel__traverse": "*" - } - }, - "node_modules/@types/babel__generator": { - "version": "7.27.0", - "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", - "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.0.0" - } - }, - "node_modules/@types/babel__template": { - "version": "7.4.4", - "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", - "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.1.0", - "@babel/types": "^7.0.0" - } - }, - "node_modules/@types/babel__traverse": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", - "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.28.2" - } - }, - "node_modules/@types/graceful-fs": { - "version": "4.1.9", - "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", - "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/istanbul-lib-coverage": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", - "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/istanbul-lib-report": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", - "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/istanbul-lib-coverage": "*" - } - }, - "node_modules/@types/istanbul-reports": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", - "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/istanbul-lib-report": "*" - } - }, - "node_modules/@types/node": { - "version": "25.6.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.0.tgz", - "integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==", - "license": "MIT", - "dependencies": { - "undici-types": "~7.19.0" - } - }, - "node_modules/@types/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", - "license": "MIT" - }, - "node_modules/@types/stack-utils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", - "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/yargs": { - "version": "17.0.35", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.35.tgz", - "integrity": "sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/yargs-parser": "*" - } - }, - "node_modules/@types/yargs-parser": { - "version": "21.0.3", - "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", - "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/ansi-escapes": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", - "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "type-fest": "^0.21.3" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/anymatch": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", - "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, - "license": "ISC", - "dependencies": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/babel-jest": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", - "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/transform": "^29.7.0", - "@types/babel__core": "^7.1.14", - "babel-plugin-istanbul": "^6.1.1", - "babel-preset-jest": "^29.6.3", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "@babel/core": "^7.8.0" - } - }, - "node_modules/babel-plugin-istanbul": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", - "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@babel/helper-plugin-utils": "^7.0.0", - "@istanbuljs/load-nyc-config": "^1.0.0", - "@istanbuljs/schema": "^0.1.2", - "istanbul-lib-instrument": "^5.0.4", - "test-exclude": "^6.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/babel-plugin-istanbul/node_modules/istanbul-lib-instrument": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", - "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@babel/core": "^7.12.3", - "@babel/parser": "^7.14.7", - "@istanbuljs/schema": "^0.1.2", - "istanbul-lib-coverage": "^3.2.0", - "semver": "^6.3.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/babel-plugin-jest-hoist": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz", - "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.3.3", - "@babel/types": "^7.3.3", - "@types/babel__core": "^7.1.14", - "@types/babel__traverse": "^7.0.6" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/babel-preset-current-node-syntax": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.2.0.tgz", - "integrity": "sha512-E/VlAEzRrsLEb2+dv8yp3bo4scof3l9nR4lrld+Iy5NyVqgVYUJnDAmunkhPMisRI32Qc4iRiz425d8vM++2fg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/plugin-syntax-async-generators": "^7.8.4", - "@babel/plugin-syntax-bigint": "^7.8.3", - "@babel/plugin-syntax-class-properties": "^7.12.13", - "@babel/plugin-syntax-class-static-block": "^7.14.5", - "@babel/plugin-syntax-import-attributes": "^7.24.7", - "@babel/plugin-syntax-import-meta": "^7.10.4", - "@babel/plugin-syntax-json-strings": "^7.8.3", - "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", - "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", - "@babel/plugin-syntax-numeric-separator": "^7.10.4", - "@babel/plugin-syntax-object-rest-spread": "^7.8.3", - "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", - "@babel/plugin-syntax-optional-chaining": "^7.8.3", - "@babel/plugin-syntax-private-property-in-object": "^7.14.5", - "@babel/plugin-syntax-top-level-await": "^7.14.5" - }, - "peerDependencies": { - "@babel/core": "^7.0.0 || ^8.0.0-0" - } - }, - "node_modules/babel-preset-jest": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz", - "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==", - "dev": true, - "license": "MIT", - "dependencies": { - "babel-plugin-jest-hoist": "^29.6.3", - "babel-preset-current-node-syntax": "^1.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, - "license": "MIT" - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/baseline-browser-mapping": { - "version": "2.10.21", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.21.tgz", - "integrity": "sha512-Q+rUQ7Uz8AHM7DEaNdwvfFCTq7a43lNTzuS94eiWqwyxfV/wJv+oUivef51T91mmRY4d4A1u9rcSvkeufCVXlA==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "baseline-browser-mapping": "dist/cli.cjs" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/bignumber.js": { - "version": "9.3.1", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", - "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/brace-expansion": { - "version": "1.1.14", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", - "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "license": "MIT", - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/browserslist": { - "version": "4.28.2", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", - "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "baseline-browser-mapping": "^2.10.12", - "caniuse-lite": "^1.0.30001782", - "electron-to-chromium": "^1.5.328", - "node-releases": "^2.0.36", - "update-browserslist-db": "^1.2.3" - }, - "bin": { - "browserslist": "cli.js" - }, - "engines": { - "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" - } - }, - "node_modules/bser": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", - "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "node-int64": "^0.4.0" - } - }, - "node_modules/buffer-equal-constant-time": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", - "license": "BSD-3-Clause" - }, - "node_modules/buffer-from": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/caniuse-lite": { - "version": "1.0.30001790", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001790.tgz", - "integrity": "sha512-bOoxfJPyYo+ds6W0YfptaCWbFnJYjh2Y1Eow5lRv+vI2u8ganPZqNm1JwNh0t2ELQCqIWg4B3dWEusgAmsoyOw==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/caniuse-lite" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "CC-BY-4.0" - }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/char-regex": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", - "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/ci-info": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", - "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/cjs-module-lexer": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", - "integrity": "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/co": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", - "dev": true, - "license": "MIT", - "engines": { - "iojs": ">= 1.0.0", - "node": ">= 0.12.0" - } - }, - "node_modules/collect-v8-coverage": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.3.tgz", - "integrity": "sha512-1L5aqIkwPfiodaMgQunkF1zRhNqifHBmtbbbxcr6yVxxBnliw4TDOW6NxpO8DJLgJ16OT+Y4ztZqP6p/FtXnAw==", - "dev": true, - "license": "MIT" - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, - "license": "MIT" - }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, - "license": "MIT" - }, - "node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true, - "license": "MIT" - }, - "node_modules/create-jest": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", - "integrity": "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "chalk": "^4.0.0", - "exit": "^0.1.2", - "graceful-fs": "^4.2.9", - "jest-config": "^29.7.0", - "jest-util": "^29.7.0", - "prompts": "^2.0.1" - }, - "bin": { - "create-jest": "bin/create-jest.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/dedent": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.2.tgz", - "integrity": "sha512-WzMx3mW98SN+zn3hgemf4OzdmyNhhhKz5Ay0pUfQiMQ3e1g+xmTJWp/pKdwKVXhdSkAEGIIzqeuWrL3mV/AXbA==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "babel-plugin-macros": "^3.1.0" - }, - "peerDependenciesMeta": { - "babel-plugin-macros": { - "optional": true - } - } - }, - "node_modules/deepmerge": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", - "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/detect-newline": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", - "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/diff-sequences": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", - "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", - "license": "Apache-2.0", - "dependencies": { - "safe-buffer": "^5.0.1" - } - }, - "node_modules/electron-to-chromium": { - "version": "1.5.344", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.344.tgz", - "integrity": "sha512-4MxfbmNDm+KPh066EZy+eUnkcDPcZ35wNmOWzFuh/ijvHsve6kbLTLURy88uCNK5FbpN+yk2nQY6BYh1GEt+wg==", - "dev": true, - "license": "ISC" - }, - "node_modules/emittery": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", - "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sindresorhus/emittery?sponsor=1" - } - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/error-ex": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", - "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-arrayish": "^0.2.1" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/escape-string-regexp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", - "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/execa": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", - "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", - "dev": true, - "license": "MIT", - "dependencies": { - "cross-spawn": "^7.0.3", - "get-stream": "^6.0.0", - "human-signals": "^2.1.0", - "is-stream": "^2.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^4.0.1", - "onetime": "^5.1.2", - "signal-exit": "^3.0.3", - "strip-final-newline": "^2.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sindresorhus/execa?sponsor=1" - } - }, - "node_modules/exit": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", - "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", - "dev": true, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/expect": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", - "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/expect-utils": "^29.7.0", - "jest-get-type": "^29.6.3", - "jest-matcher-utils": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "license": "MIT" - }, - "node_modules/fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true, - "license": "MIT" - }, - "node_modules/fb-watchman": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", - "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "bser": "2.1.1" - } - }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "license": "MIT", - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "license": "MIT", - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true, - "license": "ISC" - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/gaxios": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz", - "integrity": "sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "node-fetch": "^3.3.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/gcp-metadata": { - "version": "8.1.2", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", - "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", - "license": "Apache-2.0", - "dependencies": { - "gaxios": "^7.0.0", - "google-logging-utils": "^1.0.0", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/gensync": { - "version": "1.0.0-beta.2", - "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", - "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/get-package-type": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", - "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/get-stream": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", - "dev": true, - "license": "ISC", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/google-auth-library": { - "version": "10.6.2", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.6.2.tgz", - "integrity": "sha512-e27Z6EThmVNNvtYASwQxose/G57rkRuaRbQyxM2bvYLLX/GqWZ5chWq2EBoUchJbCc57eC9ArzO5wMsEmWftCw==", - "license": "Apache-2.0", - "dependencies": { - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^7.1.4", - "gcp-metadata": "8.1.2", - "google-logging-utils": "1.1.3", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/google-logging-utils": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", - "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/hasown": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", - "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", - "dev": true, - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/html-escaper": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", - "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", - "dev": true, - "license": "MIT" - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/human-signals": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", - "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=10.17.0" - } - }, - "node_modules/import-local": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", - "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", - "dev": true, - "license": "MIT", - "dependencies": { - "pkg-dir": "^4.2.0", - "resolve-cwd": "^3.0.0" - }, - "bin": { - "import-local-fixture": "fixtures/cli.js" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/imurmurhash": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.19" - } - }, - "node_modules/inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", - "dev": true, - "license": "ISC", - "dependencies": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/is-arrayish": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", - "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", - "dev": true, - "license": "MIT" - }, - "node_modules/is-core-module": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", - "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", - "dev": true, - "license": "MIT", - "dependencies": { - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-generator-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", - "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true, - "license": "ISC" - }, - "node_modules/istanbul-lib-coverage": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", - "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-instrument": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", - "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@babel/core": "^7.23.9", - "@babel/parser": "^7.23.9", - "@istanbuljs/schema": "^0.1.3", - "istanbul-lib-coverage": "^3.2.0", - "semver": "^7.5.4" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-instrument/node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-report": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", - "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "istanbul-lib-coverage": "^3.0.0", - "make-dir": "^4.0.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-source-maps": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", - "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0", - "source-map": "^0.6.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-reports": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", - "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "html-escaper": "^2.0.0", - "istanbul-lib-report": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/jest": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", - "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/core": "^29.7.0", - "@jest/types": "^29.6.3", - "import-local": "^3.0.2", - "jest-cli": "^29.7.0" - }, - "bin": { - "jest": "bin/jest.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/jest-changed-files": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz", - "integrity": "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==", - "dev": true, - "license": "MIT", - "dependencies": { - "execa": "^5.0.0", - "jest-util": "^29.7.0", - "p-limit": "^3.1.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-circus": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz", - "integrity": "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/expect": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "co": "^4.6.0", - "dedent": "^1.0.0", - "is-generator-fn": "^2.0.0", - "jest-each": "^29.7.0", - "jest-matcher-utils": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-runtime": "^29.7.0", - "jest-snapshot": "^29.7.0", - "jest-util": "^29.7.0", - "p-limit": "^3.1.0", - "pretty-format": "^29.7.0", - "pure-rand": "^6.0.0", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-cli": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz", - "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/core": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/types": "^29.6.3", - "chalk": "^4.0.0", - "create-jest": "^29.7.0", - "exit": "^0.1.2", - "import-local": "^3.0.2", - "jest-config": "^29.7.0", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "yargs": "^17.3.1" - }, - "bin": { - "jest": "bin/jest.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/jest-config": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz", - "integrity": "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.11.6", - "@jest/test-sequencer": "^29.7.0", - "@jest/types": "^29.6.3", - "babel-jest": "^29.7.0", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "deepmerge": "^4.2.2", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", - "jest-circus": "^29.7.0", - "jest-environment-node": "^29.7.0", - "jest-get-type": "^29.6.3", - "jest-regex-util": "^29.6.3", - "jest-resolve": "^29.7.0", - "jest-runner": "^29.7.0", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "micromatch": "^4.0.4", - "parse-json": "^5.2.0", - "pretty-format": "^29.7.0", - "slash": "^3.0.0", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "@types/node": "*", - "ts-node": ">=9.0.0" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "ts-node": { - "optional": true - } - } - }, - "node_modules/jest-diff": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz", - "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "diff-sequences": "^29.6.3", - "jest-get-type": "^29.6.3", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-docblock": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz", - "integrity": "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==", - "dev": true, - "license": "MIT", - "dependencies": { - "detect-newline": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-each": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz", - "integrity": "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "chalk": "^4.0.0", - "jest-get-type": "^29.6.3", - "jest-util": "^29.7.0", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-environment-node": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz", - "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/fake-timers": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "jest-mock": "^29.7.0", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-get-type": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", - "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-haste-map": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz", - "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@types/graceful-fs": "^4.1.3", - "@types/node": "*", - "anymatch": "^3.0.3", - "fb-watchman": "^2.0.0", - "graceful-fs": "^4.2.9", - "jest-regex-util": "^29.6.3", - "jest-util": "^29.7.0", - "jest-worker": "^29.7.0", - "micromatch": "^4.0.4", - "walker": "^1.0.8" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "optionalDependencies": { - "fsevents": "^2.3.2" - } - }, - "node_modules/jest-leak-detector": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz", - "integrity": "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==", - "dev": true, - "license": "MIT", - "dependencies": { - "jest-get-type": "^29.6.3", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-matcher-utils": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz", - "integrity": "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "jest-diff": "^29.7.0", - "jest-get-type": "^29.6.3", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-message-util": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz", - "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.12.13", - "@jest/types": "^29.6.3", - "@types/stack-utils": "^2.0.0", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "micromatch": "^4.0.4", - "pretty-format": "^29.7.0", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-mock": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz", - "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@types/node": "*", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-pnp-resolver": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", - "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - }, - "peerDependencies": { - "jest-resolve": "*" - }, - "peerDependenciesMeta": { - "jest-resolve": { - "optional": true - } - } - }, - "node_modules/jest-regex-util": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", - "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-resolve": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz", - "integrity": "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "jest-pnp-resolver": "^1.2.2", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "resolve": "^1.20.0", - "resolve.exports": "^2.0.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-resolve-dependencies": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz", - "integrity": "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==", - "dev": true, - "license": "MIT", - "dependencies": { - "jest-regex-util": "^29.6.3", - "jest-snapshot": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-runner": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz", - "integrity": "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/console": "^29.7.0", - "@jest/environment": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "emittery": "^0.13.1", - "graceful-fs": "^4.2.9", - "jest-docblock": "^29.7.0", - "jest-environment-node": "^29.7.0", - "jest-haste-map": "^29.7.0", - "jest-leak-detector": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-resolve": "^29.7.0", - "jest-runtime": "^29.7.0", - "jest-util": "^29.7.0", - "jest-watcher": "^29.7.0", - "jest-worker": "^29.7.0", - "p-limit": "^3.1.0", - "source-map-support": "0.5.13" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-runtime": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz", - "integrity": "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/fake-timers": "^29.7.0", - "@jest/globals": "^29.7.0", - "@jest/source-map": "^29.6.3", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "cjs-module-lexer": "^1.0.0", - "collect-v8-coverage": "^1.0.0", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-mock": "^29.7.0", - "jest-regex-util": "^29.6.3", - "jest-resolve": "^29.7.0", - "jest-snapshot": "^29.7.0", - "jest-util": "^29.7.0", - "slash": "^3.0.0", - "strip-bom": "^4.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-snapshot": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz", - "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.11.6", - "@babel/generator": "^7.7.2", - "@babel/plugin-syntax-jsx": "^7.7.2", - "@babel/plugin-syntax-typescript": "^7.7.2", - "@babel/types": "^7.3.3", - "@jest/expect-utils": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "babel-preset-current-node-syntax": "^1.0.0", - "chalk": "^4.0.0", - "expect": "^29.7.0", - "graceful-fs": "^4.2.9", - "jest-diff": "^29.7.0", - "jest-get-type": "^29.6.3", - "jest-matcher-utils": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0", - "natural-compare": "^1.4.0", - "pretty-format": "^29.7.0", - "semver": "^7.5.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/jest-util": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", - "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "graceful-fs": "^4.2.9", - "picomatch": "^2.2.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-validate": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz", - "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "camelcase": "^6.2.0", - "chalk": "^4.0.0", - "jest-get-type": "^29.6.3", - "leven": "^3.1.0", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-validate/node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/jest-watcher": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz", - "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/test-result": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "ansi-escapes": "^4.2.1", - "chalk": "^4.0.0", - "emittery": "^0.13.1", - "jest-util": "^29.7.0", - "string-length": "^4.0.1" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-worker": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", - "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "jest-util": "^29.7.0", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-worker/node_modules/supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/js-yaml": { - "version": "3.14.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", - "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/jsesc": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", - "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", - "dev": true, - "license": "MIT", - "bin": { - "jsesc": "bin/jsesc" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "license": "MIT", - "dependencies": { - "bignumber.js": "^9.0.0" - } - }, - "node_modules/json-parse-even-better-errors": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", - "dev": true, - "license": "MIT" - }, - "node_modules/json5": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", - "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "dev": true, - "license": "MIT", - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/jwa": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", - "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", - "license": "MIT", - "dependencies": { - "buffer-equal-constant-time": "^1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/jws": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", - "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", - "license": "MIT", - "dependencies": { - "jwa": "^2.0.1", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/kleur": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", - "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/leven": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", - "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/lines-and-columns": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", - "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true, - "license": "MIT" - }, - "node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/long": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", - "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", - "license": "Apache-2.0" - }, - "node_modules/lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^3.0.2" - } - }, - "node_modules/make-dir": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", - "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", - "dev": true, - "license": "MIT", - "dependencies": { - "semver": "^7.5.3" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/make-dir/node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/makeerror": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", - "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "tmpl": "1.0.5" - } - }, - "node_modules/merge-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", - "dev": true, - "license": "MIT" - }, - "node_modules/micromatch": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", - "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dev": true, - "license": "MIT", - "dependencies": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/natural-compare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true, - "license": "MIT" - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "license": "MIT", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, - "node_modules/node-int64": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", - "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", - "dev": true, - "license": "MIT" - }, - "node_modules/node-releases": { - "version": "2.0.38", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.38.tgz", - "integrity": "sha512-3qT/88Y3FbH/Kx4szpQQ4HzUbVrHPKTLVpVocKiLfoYvw9XSGOX2FmD2d6DrXbVYyAQTF2HeF6My8jmzx7/CRw==", - "dev": true, - "license": "MIT" - }, - "node_modules/normalize-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", - "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/npm-run-path": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", - "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, - "license": "ISC", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/onetime": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", - "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "dev": true, - "license": "MIT", - "dependencies": { - "mimic-fn": "^2.1.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-locate/node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-retry": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", - "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", - "license": "MIT", - "dependencies": { - "@types/retry": "0.12.0", - "retry": "^0.13.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/parse-json": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", - "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.0.0", - "error-ex": "^1.3.1", - "json-parse-even-better-errors": "^2.3.0", - "lines-and-columns": "^1.1.6" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-parse": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true, - "license": "MIT" - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", - "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/pirates": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", - "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 6" - } - }, - "node_modules/pkg-dir": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", - "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "find-up": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/pretty-format/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/prompts": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", - "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "kleur": "^3.0.3", - "sisteransi": "^1.0.5" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/protobufjs": { - "version": "7.6.0", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.0.tgz", - "integrity": "sha512-LtESOsMPTZgyYtwxhvdgdjGL0HmXEaRA/hVD6sol4zA60hVXXXP/SGmxnqDbgGE8gy7pYex7cym+5vYPcmaXBQ==", - "hasInstallScript": true, - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.5", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.1", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.2", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.1", - "@types/node": ">=13.7.0", - "long": "^5.3.2" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/pure-rand": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", - "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/dubzzz" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fast-check" - } - ], - "license": "MIT" - }, - "node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/resolve": { - "version": "1.22.12", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", - "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "is-core-module": "^2.16.1", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/resolve-cwd": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", - "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "resolve-from": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/resolve-from": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/resolve.exports": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.3.tgz", - "integrity": "sha512-OcXjMsGdhL4XnbShKpAcSqPMzQoYkYyhbEaeSko47MjRP9NfEQMhZkXL1DoFlt9LWQn4YttrdnV6X2OiyzBi+A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/sisteransi": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", - "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", - "dev": true, - "license": "MIT" - }, - "node_modules/slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/source-map-support": { - "version": "0.5.13", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", - "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", - "dev": true, - "license": "MIT", - "dependencies": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/stack-utils": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", - "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "escape-string-regexp": "^2.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/string-length": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", - "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "char-regex": "^1.0.2", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-bom": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", - "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-final-newline": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", - "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/strip-json-comments": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", - "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/supports-preserve-symlinks-flag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", - "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/test-exclude": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", - "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", - "dev": true, - "license": "ISC", - "dependencies": { - "@istanbuljs/schema": "^0.1.2", - "glob": "^7.1.4", - "minimatch": "^3.0.4" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/tmpl": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", - "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, - "node_modules/type-detect": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", - "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/type-fest": { - "version": "0.21.3", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", - "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/undici-types": { - "version": "7.19.2", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.19.2.tgz", - "integrity": "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==", - "license": "MIT" - }, - "node_modules/update-browserslist-db": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", - "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "escalade": "^3.2.0", - "picocolors": "^1.1.1" - }, - "bin": { - "update-browserslist-db": "cli.js" - }, - "peerDependencies": { - "browserslist": ">= 4.21.0" - } - }, - "node_modules/v8-to-istanbul": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", - "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", - "dev": true, - "license": "ISC", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.12", - "@types/istanbul-lib-coverage": "^2.0.1", - "convert-source-map": "^2.0.0" - }, - "engines": { - "node": ">=10.12.0" - } - }, - "node_modules/walker": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", - "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "makeerror": "1.0.12" - } - }, - "node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/write-file-atomic": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", - "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", - "dev": true, - "license": "ISC", - "dependencies": { - "imurmurhash": "^0.1.4", - "signal-exit": "^3.0.7" - }, - "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" - } - }, - "node_modules/ws": { - "version": "8.20.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.1.tgz", - "integrity": "sha512-It4dO0K5v//JtTXuPkfEOaI3uUN87iYPnqo/ZzqCoG3g8uhA66QUMs/SrM0YK7/NAu+r4LMh/9dq2A7k+rHs+w==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yallist": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", - "dev": true, - "license": "ISC" - }, - "node_modules/yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "cliui": "^8.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", - "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - } - } -} diff --git a/tests/pass_through_tests/package.json b/tests/pass_through_tests/package.json deleted file mode 100644 index 1d90b41ec07..00000000000 --- a/tests/pass_through_tests/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "litellm-pass-through-tests", - "version": "0.0.0", - "private": true, - "description": "JS pass-through tests for Vertex AI / Google AI Studio routes. CI-only; not published.", - "dependencies": { - "@google-cloud/vertexai": "1.12.0", - "@google/generative-ai": "0.24.1" - }, - "devDependencies": { - "jest": "29.7.0" - }, - "overrides": { - "@google-cloud/vertexai": { - "google-auth-library": "10.6.2" - } - } -} diff --git a/tests/pass_through_tests/test_gemini.js b/tests/pass_through_tests/test_gemini.js deleted file mode 100644 index ae1eb0c5cdb..00000000000 --- a/tests/pass_through_tests/test_gemini.js +++ /dev/null @@ -1,23 +0,0 @@ -// const { GoogleGenerativeAI } = require("@google/generative-ai"); - -// const genAI = new GoogleGenerativeAI(process.env.LITELLM_MASTER_KEY); -// const model = genAI.getGenerativeModel({ model: "gemini-1.5-flash" }); - -// const prompt = "Explain how AI works in 2 pages"; - -// async function run() { -// try { -// const result = await model.generateContentStream(prompt, { baseUrl: "http://localhost:4000/gemini" }); -// const response = await result.response; -// console.log(response.text()); -// for await (const chunk of result.stream) { -// const chunkText = chunk.text(); -// console.log(chunkText); -// process.stdout.write(chunkText); -// } -// } catch (error) { -// console.error("Error:", error); -// } -// } - -// run(); \ No newline at end of file diff --git a/tests/pass_through_tests/test_gemini_with_spend.test.js b/tests/pass_through_tests/test_gemini_with_spend.test.js deleted file mode 100644 index f184bbe14e3..00000000000 --- a/tests/pass_through_tests/test_gemini_with_spend.test.js +++ /dev/null @@ -1,86 +0,0 @@ -const { GoogleGenerativeAI } = require("@google/generative-ai"); -const fs = require('fs'); -const path = require('path'); - -const masterKey = process.env.LITELLM_MASTER_KEY; -if (!masterKey) throw new Error("LITELLM_MASTER_KEY must be set"); - -// Import fetch if the SDK uses it -const originalFetch = global.fetch || require('node-fetch'); - -let lastCallId; - -// Monkey-patch the fetch used internally -global.fetch = async function patchedFetch(url, options) { - const response = await originalFetch(url, options); - - // Store the call ID if it exists - lastCallId = response.headers.get('x-litellm-call-id'); - - return response; -}; - -// Configure Jest to retry flaky tests (useful for external API flakiness) -jest.retryTimes(3); - -describe('Gemini AI Tests', () => { - test('should successfully generate streaming content with tags', async () => { - const genAI = new GoogleGenerativeAI(masterKey); - - const requestOptions = { - baseUrl: 'http://127.0.0.1:4000/gemini', - customHeaders: { - "tags": "gemini-js-sdk,pass-through-endpoint" - } - }; - - const model = genAI.getGenerativeModel({ - model: 'gemini-3.1-flash-lite' - }, requestOptions); - - const prompt = 'Say "hello test" and nothing else'; - - const streamingResult = await model.generateContentStream(prompt); - expect(streamingResult).toBeDefined(); - - for await (const chunk of streamingResult.stream) { - console.log('stream chunk:', JSON.stringify(chunk)); - expect(chunk).toBeDefined(); - } - - const aggregatedResponse = await streamingResult.response; - console.log('aggregated response:', JSON.stringify(aggregatedResponse)); - expect(aggregatedResponse).toBeDefined(); - - // Use the captured callId - const callId = lastCallId; - console.log("Captured Call ID:", callId); - - // Poll for spend data with retries (DB writes can be slow in CI) - let spendData = null; - for (let attempt = 0; attempt < 6; attempt++) { - await new Promise(resolve => setTimeout(resolve, 10000)); - const spendResponse = await fetch( - `http://127.0.0.1:4000/spend/logs?request_id=${callId}`, - { headers: { 'Authorization': `Bearer ${masterKey}` } } - ); - spendData = await spendResponse.json(); - console.log(`spendData (attempt ${attempt + 1}):`, spendData); - if (spendData && spendData.length > 0 && spendData[0] && spendData[0].request_id) break; - } - - if (!spendData || !spendData.length || !spendData[0] || !spendData[0].request_id) { - console.warn('Spend data not available after polling - skipping spend assertions (DB write may be slow in CI)'); - return; - } - - expect(spendData).toBeDefined(); - expect(spendData[0].request_id).toBe(callId); - expect(spendData[0].call_type).toBe('pass_through_endpoint'); - expect(spendData[0].request_tags).toEqual(['gemini-js-sdk', 'pass-through-endpoint']); - expect(spendData[0].metadata).toHaveProperty('user_api_key'); - expect(spendData[0].model).toContain('gemini'); - expect(spendData[0].spend).toBeGreaterThan(0); - expect(spendData[0].custom_llm_provider).toBe('gemini'); - }, 90000); -}); diff --git a/tests/pass_through_tests/test_local_gemini.js b/tests/pass_through_tests/test_local_gemini.js deleted file mode 100644 index 157121e6caf..00000000000 --- a/tests/pass_through_tests/test_local_gemini.js +++ /dev/null @@ -1,58 +0,0 @@ -const { GoogleGenerativeAI, ModelParams, RequestOptions } = require("@google/generative-ai"); - -const masterKey = process.env.LITELLM_MASTER_KEY; -if (!masterKey) throw new Error("LITELLM_MASTER_KEY must be set"); - -const modelParams = { - model: 'gemini-3.1-flash-lite', -}; - -const requestOptions = { - baseUrl: 'http://127.0.0.1:4000/gemini', - customHeaders: { - "tags": "gemini-js-sdk,gemini-3.1-flash-lite" - } -}; - -const genAI = new GoogleGenerativeAI(masterKey); -const model = genAI.getGenerativeModel(modelParams, requestOptions); - -const testPrompt = "Explain how AI works"; - -async function main() { - console.log("making request") - try { - const result = await model.generateContent(testPrompt); - console.log(result.response.text()); - } catch (error) { - console.error('Error details:', { - name: error.name, - message: error.message, - cause: error.cause, - // Check if there's a network error - isNetworkError: error instanceof TypeError && error.message === 'fetch failed' - }); - - // Check if the server is running - if (error instanceof TypeError && error.message === 'fetch failed') { - console.error('Make sure your local server is running at http://localhost:4000'); - } - } -} - - -async function main_streaming() { - try { - const streamingResult = await model.generateContentStream(testPrompt); - for await (const item of streamingResult.stream) { - console.log('stream chunk: ', JSON.stringify(item)); - } - const aggregatedResponse = await streamingResult.response; - console.log('aggregated response: ', JSON.stringify(aggregatedResponse)); - } catch (error) { - console.error('Error details:', error); - } -} - -// main(); -main_streaming(); \ No newline at end of file diff --git a/tests/pass_through_tests/test_local_vertex.js b/tests/pass_through_tests/test_local_vertex.js deleted file mode 100644 index 37f8245a638..00000000000 --- a/tests/pass_through_tests/test_local_vertex.js +++ /dev/null @@ -1,46 +0,0 @@ -const { VertexAI, RequestOptions } = require('@google-cloud/vertexai'); - -const masterKey = process.env.LITELLM_MASTER_KEY; -if (!masterKey) throw new Error("LITELLM_MASTER_KEY must be set"); - - - -const vertexAI = new VertexAI({ - project: 'litellm-ci-cd', - location: 'global', - apiEndpoint: "127.0.0.1:4000/vertex-ai" -}); - -// Create customHeaders using Headers -const customHeaders = new Headers({ - "X-Litellm-Api-Key": masterKey, - tags: "vertexjs,test-2" -}); - -// Use customHeaders in RequestOptions -const requestOptions = { - customHeaders: customHeaders, -}; - -const generativeModel = vertexAI.getGenerativeModel( - { model: 'gemini-3.1-flash-lite' }, - requestOptions -); - -async function testModel() { - try { - const request = { - contents: [{role: 'user', parts: [{text: 'How are you doing today tell me your name?'}]}], - }; - const streamingResult = await generativeModel.generateContentStream(request); - for await (const item of streamingResult.stream) { - console.log('stream chunk: ', JSON.stringify(item)); - } - const aggregatedResponse = await streamingResult.response; - console.log('aggregated response: ', JSON.stringify(aggregatedResponse)); - } catch (error) { - console.error('Error:', error); - } -} - -testModel(); \ No newline at end of file diff --git a/tests/pass_through_tests/test_vertex.test.js b/tests/pass_through_tests/test_vertex.test.js deleted file mode 100644 index ec83ccdcbc0..00000000000 --- a/tests/pass_through_tests/test_vertex.test.js +++ /dev/null @@ -1,147 +0,0 @@ -const { VertexAI, RequestOptions } = require('@google-cloud/vertexai'); -const fs = require('fs'); -const path = require('path'); -const os = require('os'); -const { writeFileSync } = require('fs'); - - -// Import fetch if the SDK uses it -const originalFetch = global.fetch || require('node-fetch'); - -const { runVertexRequestOrSkip } = require('./vertex_test_helpers'); - -const masterKey = process.env.LITELLM_MASTER_KEY; -if (!masterKey) throw new Error("LITELLM_MASTER_KEY must be set"); - -// Monkey-patch the fetch used internally -global.fetch = async function patchedFetch(url, options) { - // Modify the URL to use HTTP instead of HTTPS - if (url.startsWith('https://localhost:4000')) { - url = url.replace('https://', 'http://'); - } - console.log('Patched fetch sending request to:', url); - return originalFetch(url, options); -}; - -function loadVertexAiCredentials() { - console.log("loading vertex ai credentials"); - const filepath = path.dirname(__filename); - const vertexKeyPath = path.join(filepath, "vertex_key.json"); - - // Initialize default empty service account data - let serviceAccountKeyData = {}; - - // Try to read existing vertex_key.json - try { - const content = fs.readFileSync(vertexKeyPath, 'utf8'); - if (content && content.trim()) { - serviceAccountKeyData = JSON.parse(content); - } - } catch (error) { - // File doesn't exist or is invalid, continue with empty object - } - - // Update with environment variables - const privateKeyId = process.env.VERTEX_AI_PRIVATE_KEY_ID || ""; - const privateKey = (process.env.VERTEX_AI_PRIVATE_KEY || "").replace(/\\n/g, "\n"); - - serviceAccountKeyData.private_key_id = privateKeyId; - serviceAccountKeyData.private_key = privateKey; - - // Create temporary file - const tempFilePath = path.join(os.tmpdir(), `vertex-credentials-${Date.now()}.json`); - writeFileSync(tempFilePath, JSON.stringify(serviceAccountKeyData, null, 2)); - - // Set environment variable - process.env.GOOGLE_APPLICATION_CREDENTIALS = tempFilePath; -} - -// Run credential loading before tests -beforeAll(() => { - loadVertexAiCredentials(); -}); - -// Configure Jest to retry flaky tests up to 3 times (useful for 429 rate limiting) -jest.retryTimes(3); - -// Non-streaming Vertex generateContent can exceed 5s in CI / under load -const VERTEX_TEST_TIMEOUT_MS = 30000; - -describe('Vertex AI Tests', () => { - test( - 'should successfully generate content from Vertex AI', - async () => { - const vertexAI = new VertexAI({ - project: 'litellm-ci-cd', - location: 'global', - apiEndpoint: "localhost:4000/vertex-ai" - }); - - const customHeaders = new Headers({ - "x-litellm-api-key": masterKey - }); - - const requestOptions = { - customHeaders: customHeaders - }; - - const generativeModel = vertexAI.getGenerativeModel( - { model: 'gemini-3.1-flash-lite' }, - requestOptions - ); - - const request = { - contents: [{role: 'user', parts: [{text: 'How are you doing today tell me your name?'}]}], - }; - - const streamingResult = await runVertexRequestOrSkip(() => - generativeModel.generateContentStream(request) - ); - if (streamingResult === null) { - return; - } - - // Add some assertions - expect(streamingResult).toBeDefined(); - - for await (const item of streamingResult.stream) { - console.log('stream chunk:', JSON.stringify(item)); - expect(item).toBeDefined(); - } - - const aggregatedResponse = await streamingResult.response; - console.log('aggregated response:', JSON.stringify(aggregatedResponse)); - expect(aggregatedResponse).toBeDefined(); - }, - VERTEX_TEST_TIMEOUT_MS - ); - - test( - 'should successfully generate non-streaming content from Vertex AI', - async () => { - const vertexAI = new VertexAI({ - project: 'litellm-ci-cd', - location: 'global', - apiEndpoint: "localhost:4000/vertex-ai" - }); - const customHeaders = new Headers({"x-litellm-api-key": masterKey}); - const requestOptions = {customHeaders: customHeaders}; - const generativeModel = vertexAI.getGenerativeModel( - {model: 'gemini-3.1-flash-lite'}, - requestOptions - ); - const request = {contents: [{role: 'user', parts: [{text: 'What is 2+2?'}]}]}; - - const result = await runVertexRequestOrSkip(() => - generativeModel.generateContent(request) - ); - if (result === null) { - return; - } - expect(result).toBeDefined(); - expect(result.response).toBeDefined(); - console.log('non-streaming response:', JSON.stringify(result.response)); - }, - VERTEX_TEST_TIMEOUT_MS - ); -}); diff --git a/tests/pass_through_tests/test_vertex_with_spend.test.js b/tests/pass_through_tests/test_vertex_with_spend.test.js deleted file mode 100644 index 57d697185ea..00000000000 --- a/tests/pass_through_tests/test_vertex_with_spend.test.js +++ /dev/null @@ -1,216 +0,0 @@ -const { VertexAI, RequestOptions } = require('@google-cloud/vertexai'); -const fs = require('fs'); -const path = require('path'); -const os = require('os'); -const { writeFileSync } = require('fs'); - - -// Import fetch if the SDK uses it -const originalFetch = global.fetch || require('node-fetch'); - -let lastCallId; - -const { runVertexRequestOrSkip } = require('./vertex_test_helpers'); - -const masterKey = process.env.LITELLM_MASTER_KEY; -if (!masterKey) throw new Error("LITELLM_MASTER_KEY must be set"); - -// Monkey-patch the fetch used internally -global.fetch = async function patchedFetch(url, options) { - // Modify the URL to use HTTP instead of HTTPS - if (url.startsWith('https://127.0.0.1:4000')) { - url = url.replace('https://', 'http://'); - } - console.log('Patched fetch sending request to:', url); - - const response = await originalFetch(url, options); - - // Store the call ID if it exists - lastCallId = response.headers.get('x-litellm-call-id'); - - return response; -}; - -function loadVertexAiCredentials() { - console.log("loading vertex ai credentials"); - const filepath = path.dirname(__filename); - const vertexKeyPath = path.join(filepath, "vertex_key.json"); - - // Initialize default empty service account data - let serviceAccountKeyData = {}; - - // Try to read existing vertex_key.json - try { - const content = fs.readFileSync(vertexKeyPath, 'utf8'); - if (content && content.trim()) { - serviceAccountKeyData = JSON.parse(content); - } - } catch (error) { - // File doesn't exist or is invalid, continue with empty object - } - - // Update with environment variables - const privateKeyId = process.env.VERTEX_AI_PRIVATE_KEY_ID || ""; - const privateKey = (process.env.VERTEX_AI_PRIVATE_KEY || "").replace(/\\n/g, "\n"); - - serviceAccountKeyData.private_key_id = privateKeyId; - serviceAccountKeyData.private_key = privateKey; - - // Create temporary file - const tempFilePath = path.join(os.tmpdir(), `vertex-credentials-${Date.now()}.json`); - writeFileSync(tempFilePath, JSON.stringify(serviceAccountKeyData, null, 2)); - - // Set environment variable - process.env.GOOGLE_APPLICATION_CREDENTIALS = tempFilePath; -} - -// Run credential loading before tests -beforeAll(() => { - loadVertexAiCredentials(); -}); - -// Configure Jest to retry flaky tests up to 3 times (useful for 429 rate limiting) -jest.retryTimes(3); - -describe('Vertex AI Tests', () => { - test('should successfully generate non-streaming content with tags', async () => { - const vertexAI = new VertexAI({ - project: 'litellm-ci-cd', - location: 'global', - apiEndpoint: "127.0.0.1:4000/vertex_ai" - }); - - const customHeaders = new Headers({ - "x-litellm-api-key": masterKey, - "tags": "vertex-js-sdk,pass-through-endpoint" - }); - - const requestOptions = { - customHeaders: customHeaders - }; - - const generativeModel = vertexAI.getGenerativeModel( - { model: 'gemini-3.1-flash-lite' }, - requestOptions - ); - - const request = { - contents: [{role: 'user', parts: [{text: 'Say "hello test" and nothing else'}]}] - }; - - const result = await runVertexRequestOrSkip(() => - generativeModel.generateContent(request) - ); - if (result === null) { - return; - } - expect(result).toBeDefined(); - - // Use the captured callId - const callId = lastCallId; - console.log("Captured Call ID:", callId); - - // Poll for spend data with retries (DB writes can be slow in CI) - let spendData = null; - for (let attempt = 0; attempt < 6; attempt++) { - await new Promise(resolve => setTimeout(resolve, 10000)); - const spendResponse = await fetch( - `http://127.0.0.1:4000/spend/logs?request_id=${callId}`, - { headers: { 'Authorization': `Bearer ${masterKey}` } } - ); - spendData = await spendResponse.json(); - console.log(`spendData (attempt ${attempt + 1}):`, spendData); - if (spendData && spendData.length > 0 && spendData[0] && spendData[0].request_id) break; - } - - if (!spendData || !spendData.length || !spendData[0] || !spendData[0].request_id) { - console.warn('Spend data not available after polling - skipping spend assertions (DB write may be slow in CI)'); - return; - } - - expect(spendData[0].request_id).toBe(callId); - expect(spendData[0].call_type).toBe('pass_through_endpoint'); - expect(spendData[0].request_tags).toEqual(['vertex-js-sdk', 'pass-through-endpoint']); - expect(spendData[0].metadata).toHaveProperty('user_api_key'); - expect(spendData[0].model).toContain('gemini'); - expect(spendData[0].spend).toBeGreaterThan(0); - expect(spendData[0].custom_llm_provider).toBe('vertex_ai'); - }, 90000); - - test('should successfully generate streaming content with tags', async () => { - const vertexAI = new VertexAI({ - project: 'litellm-ci-cd', - location: 'global', - apiEndpoint: "127.0.0.1:4000/vertex_ai" - }); - - const customHeaders = new Headers({ - "x-litellm-api-key": masterKey, - "tags": "vertex-js-sdk,pass-through-endpoint" - }); - - const requestOptions = { - customHeaders: customHeaders - }; - - const generativeModel = vertexAI.getGenerativeModel( - { model: 'gemini-3.1-flash-lite' }, - requestOptions - ); - - const request = { - contents: [{role: 'user', parts: [{text: 'Say "hello test" and nothing else'}]}] - }; - - const streamingResult = await runVertexRequestOrSkip(() => - generativeModel.generateContentStream(request) - ); - if (streamingResult === null) { - return; - } - expect(streamingResult).toBeDefined(); - - - // Add some assertions - expect(streamingResult).toBeDefined(); - - for await (const item of streamingResult.stream) { - console.log('stream chunk:', JSON.stringify(item)); - expect(item).toBeDefined(); - } - - const aggregatedResponse = await streamingResult.response; - console.log('aggregated response:', JSON.stringify(aggregatedResponse)); - expect(aggregatedResponse).toBeDefined(); - - // Use the captured callId - const callId = lastCallId; - console.log("Captured Call ID:", callId); - - // Poll for spend data with retries (DB writes can be slow in CI) - let spendData = null; - for (let attempt = 0; attempt < 6; attempt++) { - await new Promise(resolve => setTimeout(resolve, 10000)); - const spendResponse = await fetch( - `http://127.0.0.1:4000/spend/logs?request_id=${callId}`, - { headers: { 'Authorization': `Bearer ${masterKey}` } } - ); - spendData = await spendResponse.json(); - console.log(`spendData (attempt ${attempt + 1}):`, spendData); - if (spendData && spendData.length > 0 && spendData[0] && spendData[0].request_id) break; - } - - if (!spendData || !spendData.length || !spendData[0] || !spendData[0].request_id) { - console.warn('Spend data not available after polling - skipping spend assertions (DB write may be slow in CI)'); - return; - } - - expect(spendData[0].request_id).toBe(callId); - expect(spendData[0].call_type).toBe('pass_through_endpoint'); - expect(spendData[0].request_tags).toEqual(['vertex-js-sdk', 'pass-through-endpoint']); - expect(spendData[0].metadata).toHaveProperty('user_api_key'); - expect(spendData[0].model).toContain('gemini'); - expect(spendData[0].spend).toBeGreaterThan(0); - expect(spendData[0].custom_llm_provider).toBe('vertex_ai'); - }, 90000); -}); diff --git a/tests/pass_through_tests/vertex_key.json b/tests/pass_through_tests/vertex_key.json deleted file mode 100644 index 800969fb305..00000000000 --- a/tests/pass_through_tests/vertex_key.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "type": "service_account", - "project_id": "litellm-ci-cd", - "private_key_id": "", - "private_key": "", - "client_email": "test-litellm-ci-cd@litellm-ci-cd.iam.gserviceaccount.com", - "client_id": "116563532503305622785", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-litellm-ci-cd%40litellm-ci-cd.iam.gserviceaccount.com", - "universe_domain": "googleapis.com" -} diff --git a/tests/pass_through_tests/vertex_test_helpers.js b/tests/pass_through_tests/vertex_test_helpers.js deleted file mode 100644 index d637f20f916..00000000000 --- a/tests/pass_through_tests/vertex_test_helpers.js +++ /dev/null @@ -1,27 +0,0 @@ -function isVertexQuotaError(error) { - const message = [ - error && error.message, - error && error.stack, - error && error.cause && JSON.stringify(error.cause), - ].filter(Boolean).join('\n'); - - return ( - message.includes('429') || - message.includes('Too Many Requests') || - message.includes('RESOURCE_EXHAUSTED') - ); -} - -async function runVertexRequestOrSkip(requestFn) { - try { - return await requestFn(); - } catch (error) { - if (isVertexQuotaError(error)) { - console.warn('Vertex AI quota exhausted; skipping live provider assertions for this run'); - return null; - } - throw error; - } -} - -module.exports = { isVertexQuotaError, runVertexRequestOrSkip }; diff --git a/tests/proxy_behavior/auth/conftest.py b/tests/proxy_behavior/auth/conftest.py deleted file mode 100644 index 21982fa25cd..00000000000 --- a/tests/proxy_behavior/auth/conftest.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Session-scoped PrismaClient for auth behavior tests that run raw SQL against a real Postgres.""" - -import os -from unittest.mock import MagicMock - -import pytest -import pytest_asyncio - -from litellm.proxy.utils import PrismaClient - - -@pytest_asyncio.fixture(scope="session", loop_scope="session") -async def prisma(): - database_url = os.environ.get("DATABASE_URL") - if not database_url: - pytest.skip("DATABASE_URL not set") # test-quality-ok: this suite exists to run SQL on a real Postgres - client = PrismaClient(database_url=database_url, proxy_logging_obj=MagicMock()) - await client.connect() - yield client - await client.disconnect() diff --git a/tests/proxy_behavior/management/actors.py b/tests/proxy_behavior/management/actors.py deleted file mode 100644 index 6c2f1a61ce1..00000000000 --- a/tests/proxy_behavior/management/actors.py +++ /dev/null @@ -1,279 +0,0 @@ -"""Read-world seed for the authz matrix tests: 2 orgs, 3 teams, 9 actors.""" - -import enum -import uuid -from dataclasses import dataclass -from typing import Any, Dict - -from prisma import Json - -from litellm.proxy._types import LitellmUserRoles -from litellm.proxy.utils import PrismaClient, hash_token - - -class Actor(str, enum.Enum): - PROXY_ADMIN = "proxy_admin" - ORG_ADMIN = "org_admin" - TEAM_ADMIN = "team_admin" - INTERNAL_USER = "internal_user" - OWNER = "owner" - UNRELATED_SAME_ORG = "unrelated_same_org" - CROSS_ORG_USER = "cross_org_user" - SERVICE_ACCOUNT = "service_account" - ORG_B_ADMIN = "org_b_admin" - - -PREFIX = "behavior-pin-" -ORG_A = PREFIX + "org-a" -ORG_B = PREFIX + "org-b" -TEAM_ALPHA = PREFIX + "team-alpha" -TEAM_BETA = PREFIX + "team-beta" -TEAM_GAMMA = PREFIX + "team-gamma" -BUDGET_ID = PREFIX + "budget" - - -@dataclass(frozen=True) -class SeededKey: - user_id: str - cleartext: str - hashed: str - - -@dataclass(frozen=True) -class World: - org_a_id: str - org_b_id: str - team_alpha_id: str - team_beta_id: str - team_gamma_id: str - keys: Dict[Actor, SeededKey] - - -def _new_clear_key() -> str: - return "sk-" + uuid.uuid4().hex - - -def _actor_profile() -> Dict[Actor, Dict[str, Any]]: - return { - Actor.PROXY_ADMIN: { - "user_role": LitellmUserRoles.PROXY_ADMIN.value, - "team_id": None, - "organization_id": None, - }, - Actor.ORG_ADMIN: { - "user_role": LitellmUserRoles.ORG_ADMIN.value, - "team_id": None, - "organization_id": ORG_A, - }, - Actor.TEAM_ADMIN: { - "user_role": LitellmUserRoles.INTERNAL_USER.value, - "team_id": TEAM_ALPHA, - "organization_id": ORG_A, - }, - Actor.INTERNAL_USER: { - "user_role": LitellmUserRoles.INTERNAL_USER.value, - "team_id": TEAM_ALPHA, - "organization_id": ORG_A, - }, - Actor.OWNER: { - "user_role": LitellmUserRoles.INTERNAL_USER.value, - "team_id": TEAM_ALPHA, - "organization_id": ORG_A, - }, - Actor.UNRELATED_SAME_ORG: { - "user_role": LitellmUserRoles.INTERNAL_USER.value, - "team_id": TEAM_ALPHA, - "organization_id": ORG_A, - }, - Actor.CROSS_ORG_USER: { - "user_role": LitellmUserRoles.INTERNAL_USER.value, - "team_id": TEAM_BETA, - "organization_id": ORG_B, - }, - Actor.SERVICE_ACCOUNT: { - "user_role": LitellmUserRoles.INTERNAL_USER.value, - "team_id": TEAM_ALPHA, - "organization_id": ORG_A, - }, - Actor.ORG_B_ADMIN: { - "user_role": LitellmUserRoles.ORG_ADMIN.value, - "team_id": None, - "organization_id": ORG_B, - }, - } - - -async def _wipe_world(prisma: PrismaClient) -> None: - await prisma.db.litellm_verificationtoken.delete_many( - where={"user_id": {"startswith": PREFIX}} - ) - await prisma.db.litellm_organizationmembership.delete_many( - where={"user_id": {"startswith": PREFIX}} - ) - await prisma.db.litellm_teammembership.delete_many( - where={"user_id": {"startswith": PREFIX}} - ) - await prisma.db.litellm_usertable.delete_many( - where={"user_id": {"startswith": PREFIX}} - ) - await prisma.db.litellm_teamtable.delete_many( - where={"team_id": {"startswith": PREFIX}} - ) - await prisma.db.litellm_organizationtable.delete_many( - where={"organization_id": {"startswith": PREFIX}} - ) - await prisma.db.litellm_budgettable.delete_many(where={"budget_id": BUDGET_ID}) - - -async def seed_world(prisma: PrismaClient) -> World: - await _wipe_world(prisma) - - await prisma.db.litellm_budgettable.create( - data={ - "budget_id": BUDGET_ID, - "created_by": "behavior-pin-seeder", - "updated_by": "behavior-pin-seeder", - } - ) - - for org_id, alias in [(ORG_A, "alpha"), (ORG_B, "beta")]: - await prisma.db.litellm_organizationtable.create( - data={ - "organization_id": org_id, - "organization_alias": alias, - "budget_id": BUDGET_ID, - "created_by": "behavior-pin-seeder", - "updated_by": "behavior-pin-seeder", - } - ) - - profiles = _actor_profile() - user_ids: Dict[Actor, str] = {actor: PREFIX + actor.value for actor in Actor} - - for actor, profile in profiles.items(): - teams_list = [profile["team_id"]] if profile["team_id"] else [] - await prisma.db.litellm_usertable.create( - data={ - "user_id": user_ids[actor], - "user_role": profile["user_role"], - "team_id": profile["team_id"], - "organization_id": profile["organization_id"], - "teams": teams_list, - } - ) - - # _get_user_in_team in key_management_endpoints.py walks members_with_roles - # (a JSON list of {user_id, role}), not the String[] members column — - # populate both to match what /team/new produces. - await prisma.db.litellm_teamtable.create( - data={ - "team_id": TEAM_ALPHA, - "team_alias": "alpha-1", - "organization_id": ORG_A, - "admins": [user_ids[Actor.TEAM_ADMIN]], - "members": [ - user_ids[Actor.TEAM_ADMIN], - user_ids[Actor.INTERNAL_USER], - user_ids[Actor.OWNER], - user_ids[Actor.UNRELATED_SAME_ORG], - user_ids[Actor.SERVICE_ACCOUNT], - ], - "members_with_roles": Json( - [ - {"user_id": user_ids[Actor.TEAM_ADMIN], "role": "admin"}, - {"user_id": user_ids[Actor.INTERNAL_USER], "role": "user"}, - {"user_id": user_ids[Actor.OWNER], "role": "user"}, - {"user_id": user_ids[Actor.UNRELATED_SAME_ORG], "role": "user"}, - {"user_id": user_ids[Actor.SERVICE_ACCOUNT], "role": "user"}, - ] - ), - } - ) - await prisma.db.litellm_teamtable.create( - data={ - "team_id": TEAM_BETA, - "team_alias": "beta-1", - "organization_id": ORG_B, - "admins": [], - "members": [user_ids[Actor.CROSS_ORG_USER]], - "members_with_roles": Json( - [ - {"user_id": user_ids[Actor.CROSS_ORG_USER], "role": "user"}, - ] - ), - } - ) - # TEAM_GAMMA: ORG_A team with no actor members — the "same-org, - # not-my-team" read target. - await prisma.db.litellm_teamtable.create( - data={ - "team_id": TEAM_GAMMA, - "team_alias": "gamma-1", - "organization_id": ORG_A, - "admins": [], - "members": [], - "members_with_roles": Json([]), - } - ) - - for actor, org_id, role in [ - (Actor.ORG_ADMIN, ORG_A, "org_admin"), - (Actor.TEAM_ADMIN, ORG_A, "internal_user"), - (Actor.INTERNAL_USER, ORG_A, "internal_user"), - (Actor.OWNER, ORG_A, "internal_user"), - (Actor.UNRELATED_SAME_ORG, ORG_A, "internal_user"), - (Actor.SERVICE_ACCOUNT, ORG_A, "internal_user"), - (Actor.CROSS_ORG_USER, ORG_B, "internal_user"), - (Actor.ORG_B_ADMIN, ORG_B, "org_admin"), - ]: - await prisma.db.litellm_organizationmembership.create( - data={ - "user_id": user_ids[actor], - "organization_id": org_id, - "user_role": role, - } - ) - - for actor, team_id in [ - (Actor.TEAM_ADMIN, TEAM_ALPHA), - (Actor.INTERNAL_USER, TEAM_ALPHA), - (Actor.OWNER, TEAM_ALPHA), - (Actor.UNRELATED_SAME_ORG, TEAM_ALPHA), - (Actor.SERVICE_ACCOUNT, TEAM_ALPHA), - (Actor.CROSS_ORG_USER, TEAM_BETA), - ]: - await prisma.db.litellm_teammembership.create( - data={"user_id": user_ids[actor], "team_id": team_id} - ) - - keys: Dict[Actor, SeededKey] = {} - for actor, profile in profiles.items(): - cleartext = _new_clear_key() - hashed = hash_token(cleartext) - token_data: Dict[str, Any] = { - "token": hashed, - "key_name": PREFIX + actor.value + "-key", - "user_id": user_ids[actor], - # LiteLLM_VerificationTokenView's models field rejects NULL even - # though the column is nullable in Postgres. - "models": [], - } - if profile["team_id"]: - token_data["team_id"] = profile["team_id"] - if profile["organization_id"]: - token_data["organization_id"] = profile["organization_id"] - if actor == Actor.SERVICE_ACCOUNT: - token_data["metadata"] = Json({"service_account_id": user_ids[actor]}) - await prisma.db.litellm_verificationtoken.create(data=token_data) - keys[actor] = SeededKey( - user_id=user_ids[actor], cleartext=cleartext, hashed=hashed - ) - - return World( - org_a_id=ORG_A, - org_b_id=ORG_B, - team_alpha_id=TEAM_ALPHA, - team_beta_id=TEAM_BETA, - team_gamma_id=TEAM_GAMMA, - keys=keys, - ) diff --git a/tests/proxy_behavior/management/conftest.py b/tests/proxy_behavior/management/conftest.py deleted file mode 100644 index 74db323e7f2..00000000000 --- a/tests/proxy_behavior/management/conftest.py +++ /dev/null @@ -1,396 +0,0 @@ -"""Session-scoped async ASGI client for HTTP-boundary behavior tests.""" - -import os -import tempfile -import uuid -from dataclasses import dataclass -from typing import Any, AsyncIterator, Dict, Optional - -import httpx -import pytest_asyncio -import yaml -from prisma import Json - -from litellm.proxy.utils import hash_token - -MASTER_KEY = "sk-proxy-behavior-master-key" -SCRATCH_PREFIX = "scratch-" - - -def _write_minimal_proxy_config() -> str: - config = { - "general_settings": {"master_key": MASTER_KEY}, - "litellm_settings": {}, - } - database_url = os.environ.get("DATABASE_URL") - if database_url: - config["general_settings"]["database_url"] = database_url - f = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) - yaml.dump(config, f) - f.close() - return f.name - - -@pytest_asyncio.fixture(scope="package") -async def proxy_app(): - from litellm.proxy import proxy_server - from litellm.proxy.proxy_server import ( - app, - cleanup_router_config_variables, - initialize, - proxy_startup_event, - ) - - cleanup_router_config_variables() - config_path = _write_minimal_proxy_config() - - # proxy_startup_event re-reads master_key from LITELLM_MASTER_KEY and - # unconditionally overwrites the global, even when initialize() already - # set it from the config YAML. Force (not setdefault) both vars: an - # ambient LITELLM_MASTER_KEY with a different value would make the proxy - # authenticate on that key while the tests still send MASTER_KEY. - os.environ["LITELLM_MASTER_KEY"] = MASTER_KEY - os.environ["CONFIG_FILE_PATH"] = config_path - - await initialize(config=config_path) - - # /key/regenerate is gated behind premium_user; flipping it lets the matrix - # pin authz behavior instead of the licensing gate. - proxy_server.premium_user = True - - async with proxy_startup_event(app): - proxy_server.premium_user = True # lifespan re-runs _license_check - # The lifespan fires check_view_exists() as a background task; on a - # fresh DB the first auth call races it and resolves user_id=None. - if proxy_server.prisma_client is not None: - await proxy_server.prisma_client.check_view_exists() - yield app - - -@pytest_asyncio.fixture(scope="package") -async def proxy_client(proxy_app) -> AsyncIterator[httpx.AsyncClient]: - transport = httpx.ASGITransport(app=proxy_app) - async with httpx.AsyncClient( - transport=transport, base_url="http://testserver" - ) as client: - yield client - - -@pytest_asyncio.fixture(scope="package") -async def prisma(proxy_app): - from litellm.proxy import proxy_server - - assert proxy_server.prisma_client is not None - return proxy_server.prisma_client - - -@pytest_asyncio.fixture(scope="package") -async def world(prisma): - from .actors import seed_world - - return await seed_world(prisma) - - -@dataclass(frozen=True) -class Scratch: - prefix: str - - def tag(self, suffix: str = "") -> str: - return f"{self.prefix}-{suffix}" if suffix else self.prefix - - -async def create_scratch_key( - proxy_client, - seeder_cleartext: str, - scratch_prefix: str, - *, - user_id: str, - team_id: Optional[str] = None, - organization_id: Optional[str] = None, - key_alias: Optional[str] = None, -) -> str: - """Seed a scratch-tagged key via /key/generate; returns its cleartext. - - Shared by the write-scenario matrices (key update/regenerate/delete). - key_alias defaults to scratch_prefix; pass a distinct scratch-prefixed - alias when a single scenario needs more than one key (/key/generate - enforces unique aliases). - """ - body: Dict[str, Any] = { - "key_alias": key_alias or scratch_prefix, - "user_id": user_id, - } - if team_id is not None: - body["team_id"] = team_id - if organization_id is not None: - body["organization_id"] = organization_id - resp = await proxy_client.post( - "/key/generate", - headers={"Authorization": f"Bearer {seeder_cleartext}"}, - json=body, - ) - assert resp.status_code == 200, f"setup failed: {resp.text}" - return resp.json()["key"] - - -async def create_scratch_team( - prisma, - team_id: str, - *, - organization_id: Optional[str] = None, - admin_user_ids: Optional[list] = None, - member_user_ids: Optional[list] = None, - team_member_permissions: Optional[list] = None, - models: Optional[list] = None, - max_budget: Optional[float] = None, - tpm_limit: Optional[int] = None, - rpm_limit: Optional[int] = None, - metadata: Optional[dict] = None, -) -> str: - """Raw-seed a scratch-tagged team row; returns its team_id. - - The target team for the team write matrices (update / member_*). Raw - prisma (not POST /team/new) avoids creation side effects — no creator - auto-add, no membership rows written onto the world's users — so seeding - never mutates the immutable read-world. The authz gates read the team's - members_with_roles JSON, so a raw-seeded team exercises them exactly as - a /team/new-created team would. team_id must start with the scratch - prefix so the `scratch` fixture reclaims the row. - - team_member_permissions / models seed the matching raw columns — needed - by the team-key-permission and team-model matrices. - - max_budget / tpm_limit / rpm_limit / metadata seed the team's own limit - columns (Phase 4 F1+F3) — they live directly on LiteLLM_TeamTable, no - budget-table relation needed. - """ - admin_user_ids = list(admin_user_ids or []) - member_user_ids = list(member_user_ids or []) - members_with_roles = [ - {"user_id": uid, "role": "admin"} for uid in admin_user_ids - ] + [{"user_id": uid, "role": "user"} for uid in member_user_ids] - data: Dict[str, Any] = { - "team_id": team_id, - "team_alias": team_id, - "admins": admin_user_ids, - "members": admin_user_ids + member_user_ids, - "members_with_roles": Json(members_with_roles), - } - if organization_id is not None: - data["organization_id"] = organization_id - if team_member_permissions is not None: - data["team_member_permissions"] = team_member_permissions - if models is not None: - data["models"] = models - if max_budget is not None: - data["max_budget"] = max_budget - if tpm_limit is not None: - data["tpm_limit"] = tpm_limit - if rpm_limit is not None: - data["rpm_limit"] = rpm_limit - if metadata is not None: - data["metadata"] = Json(metadata) - await prisma.db.litellm_teamtable.create(data=data) - return team_id - - -async def create_scratch_user( - prisma, - scratch_prefix: str, - *, - suffix: str, - user_email: Optional[str] = None, -) -> str: - """Raw-seed a scratch-tagged user row with no key; returns its user_id. - - The passive counterpart to create_scratch_actor: a user that requests are - made *about* rather than *by*, so it needs no verification token. The - user_id matches Scratch.tag(suffix), so the teardown reclaims it. - - A member named by user_id must already exist for a non-proxy-admin caller - — `_validate_member_user_id_provisioning` 403s a user_id with no user row - — so an authz matrix targeting a member has to seed one, or every - non-proxy-admin row 403s on provisioning and the permission gate under - test goes unexercised. - """ - user_id = f"{scratch_prefix}-{suffix}" - data: Dict[str, Any] = {"user_id": user_id, "user_role": "internal_user"} - if user_email is not None: - data["user_email"] = user_email - await prisma.db.litellm_usertable.create(data=data) - return user_id - - -async def create_scratch_org( - prisma, - scratch_prefix: str, - *, - max_budget: Optional[float] = None, - tpm_limit: Optional[int] = None, - rpm_limit: Optional[int] = None, - models: Optional[list] = None, - metadata: Optional[dict] = None, - suffix: str = "org", -) -> str: - """Seed a scratch-tagged org + its own budget row; returns organization_id. - - The org's `budget_id` points at a fresh `litellm_budgettable` row that - carries the per-org limits (`_check_org_key_limits` and the team budget - helpers read `org_table.litellm_budget_table.`, not columns on the - org row itself). Both rows share the scratch prefix so the teardown - reclaims them — budget by `budget_id` prefix (already swept), org by - `organization_id` prefix (added in this PR to the `scratch` fixture). - - models / metadata seed the matching org columns; `_check_org_team_limits` - (F3) reads `org_table.models`, and the org metadata mirror of - model_rpm_limit / model_tpm_limit is what F1's model-specific org guard - consults. - """ - org_id = f"{scratch_prefix}-{suffix}" - budget_id = f"{scratch_prefix}-{suffix}-budget" - budget_data: Dict[str, Any] = { - "budget_id": budget_id, - "created_by": "phase4-scratch", - "updated_by": "phase4-scratch", - } - if max_budget is not None: - budget_data["max_budget"] = max_budget - if tpm_limit is not None: - budget_data["tpm_limit"] = tpm_limit - if rpm_limit is not None: - budget_data["rpm_limit"] = rpm_limit - await prisma.db.litellm_budgettable.create(data=budget_data) - - org_data: Dict[str, Any] = { - "organization_id": org_id, - "organization_alias": org_id, - "budget_id": budget_id, - "created_by": "phase4-scratch", - "updated_by": "phase4-scratch", - } - if models is not None: - org_data["models"] = models - if metadata is not None: - org_data["metadata"] = Json(metadata) - await prisma.db.litellm_organizationtable.create(data=org_data) - return org_id - - -@dataclass(frozen=True) -class SeededActor: - user_id: str - cleartext: str - hashed: str - - -async def create_scratch_actor( - prisma, - scratch_prefix: str, - *, - user_role: str, - org_admin_of: tuple = (), - organization_id: Optional[str] = None, - suffix: str = "actor", -) -> SeededActor: - """Mint a scratch-prefixed user + verification token (+ org memberships). - - Reclaimed by the existing `scratch` teardown, which sweeps - litellm_usertable, litellm_verificationtoken, and - litellm_organizationmembership by scratch prefix — no bespoke cleanup - needed. Does NOT write litellm_teammembership against world teams: the - teardown reclaims that table only by team_id prefix, so a scratch actor - needing team membership must join a scratch team instead. The cleartext - is hashed with the real hash_token so the key authenticates end-to-end; - models=[] satisfies LiteLLM_VerificationTokenView. - """ - user_id = f"{scratch_prefix}-{suffix}" - cleartext = "sk-" + uuid.uuid4().hex - hashed = hash_token(cleartext) - await prisma.db.litellm_usertable.create( - data={ - "user_id": user_id, - "user_role": user_role, - "organization_id": organization_id, - } - ) - token_data: Dict[str, Any] = { - "token": hashed, - "key_name": f"{scratch_prefix}-{suffix}-key", - "key_alias": f"{scratch_prefix}-{suffix}-alias", - "user_id": user_id, - "models": [], - } - if organization_id is not None: - token_data["organization_id"] = organization_id - await prisma.db.litellm_verificationtoken.create(data=token_data) - for org_id in org_admin_of: - await prisma.db.litellm_organizationmembership.create( - data={ - "user_id": user_id, - "organization_id": org_id, - "user_role": "org_admin", - } - ) - return SeededActor(user_id=user_id, cleartext=cleartext, hashed=hashed) - - -@pytest_asyncio.fixture -async def scratch(prisma): - handle = Scratch(prefix=f"{SCRATCH_PREFIX}{uuid.uuid4().hex[:12]}") - try: - yield handle - finally: - # Children before parents to avoid FK violations. - await prisma.db.litellm_verificationtoken.delete_many( - where={ - "OR": [ - {"key_alias": {"startswith": handle.prefix}}, - {"key_name": {"startswith": handle.prefix}}, - ] - } - ) - await prisma.db.litellm_teammembership.delete_many( - where={"team_id": {"startswith": handle.prefix}} - ) - await prisma.db.litellm_organizationmembership.delete_many( - where={"user_id": {"startswith": handle.prefix}} - ) - await prisma.db.litellm_teamtable.delete_many( - where={"team_id": {"startswith": handle.prefix}} - ) - # Inviting a member by user_email allocates the user_id server-side - # (a uuid), so a scratch-prefixed email is the only handle on that - # row — sweep both, matching the token sweep above. - await prisma.db.litellm_usertable.delete_many( - where={ - "OR": [ - {"user_id": {"startswith": handle.prefix}}, - {"user_email": {"startswith": handle.prefix}}, - ] - } - ) - # F1+F3 seed scratch orgs via create_scratch_org; the world seeder is - # the only other writer of LiteLLM_OrganizationTable and uses the - # behavior-pin- prefix, so a scratch-prefixed sweep here cannot - # collide with the read-world. Org must be reclaimed BEFORE its - # budget — org.budget_id → budget.budget_id, so deleting the parent - # first would FK-violate on any still-attached scratch org. - await prisma.db.litellm_organizationtable.delete_many( - where={"organization_id": {"startswith": handle.prefix}} - ) - await prisma.db.litellm_budgettable.delete_many( - where={"budget_id": {"startswith": handle.prefix}} - ) - # /team/member_add writes LiteLLM_UserTable.teams; the available-team - # self-join writes it on a world actor whose row must survive. Strip - # dangling scratch-team refs so the read-world stays immutable. - polluted = await prisma.db.litellm_usertable.find_many( - where={"teams": {"isEmpty": False}} - ) - for user in polluted: - cleaned = [t for t in user.teams if not t.startswith(handle.prefix)] - if cleaned != list(user.teams): - await prisma.db.litellm_usertable.update( - where={"user_id": user.user_id}, - data={"teams": {"set": cleaned}}, - ) diff --git a/tests/proxy_behavior/management/test_no_management_imports.py b/tests/proxy_behavior/management/test_no_management_imports.py deleted file mode 100644 index f8c52a1c37e..00000000000 --- a/tests/proxy_behavior/management/test_no_management_imports.py +++ /dev/null @@ -1,46 +0,0 @@ -import pathlib -import re - -REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] -BEHAVIOR_DIR = REPO_ROOT / "tests" / "proxy_behavior" - -FORBIDDEN_IMPORT = re.compile(r"^\s*from\s+litellm\.proxy\.management_endpoints\b") -FORBIDDEN_AUTH_MOCK = re.compile( - r"(?:mock\.[A-Za-z_]+|patch[a-z_]*)\([^)]*user_api_key_auth" -) -# This file is the only place the forbidden patterns appear as regex source; -# exclude it so it can describe what it forbids. -SELF = pathlib.Path(__file__).resolve() - - -def _iter_py_files(): - for path in BEHAVIOR_DIR.rglob("*.py"): - if path.resolve() != SELF: - yield path - - -def _scan(pattern): - violations = [] - for path in _iter_py_files(): - for lineno, line in enumerate(path.read_text().splitlines(), start=1): - if pattern.search(line): - violations.append( - f"{path.relative_to(REPO_ROOT)}:{lineno}: {line.strip()}" - ) - return violations - - -def test_no_management_endpoint_imports(): - violations = _scan(FORBIDDEN_IMPORT) - assert not violations, ( - "tests/proxy_behavior/ must not import from litellm.proxy.management_endpoints. " - "Violations:\n " + "\n ".join(violations) - ) - - -def test_no_user_api_key_auth_mocking(): - violations = _scan(FORBIDDEN_AUTH_MOCK) - assert not violations, ( - "tests/proxy_behavior/ must not mock user_api_key_auth. " - "Violations:\n " + "\n ".join(violations) - ) diff --git a/tests/proxy_behavior/management/test_route_coverage.py b/tests/proxy_behavior/management/test_route_coverage.py deleted file mode 100644 index 1139e251a59..00000000000 --- a/tests/proxy_behavior/management/test_route_coverage.py +++ /dev/null @@ -1,91 +0,0 @@ -"""PR3.M1 — codified route coverage. - -Every route declared in the two management-endpoint source files must be -exercised by at least one behavior-suite scenario. This is a permanent -regression guard: a future route added without a behavior test fails CI here, -the same way test_no_management_imports.py codifies the G3 import grep. -""" - -import ast -import pathlib -import re - -REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] -SOURCE_FILES = [ - REPO_ROOT / "litellm/proxy/management_endpoints/key_management_endpoints.py", - REPO_ROOT / "litellm/proxy/management_endpoints/team_endpoints.py", -] -TEST_DIR = pathlib.Path(__file__).resolve().parent -SELF = pathlib.Path(__file__).resolve() - -# Captures the route literal from `@router.(""` — `\s*` spans -# newlines so multi-line decorators are matched too. -_ROUTE_DECORATOR = re.compile( - r"@router\.(?:get|post|put|delete|patch)\(\s*[\"']([^\"']+)[\"']" -) - - -def _source_routes() -> set: - routes: set = set() - for path in SOURCE_FILES: - routes.update(_ROUTE_DECORATOR.findall(path.read_text())) - return routes - - -def _route_to_regex(route: str) -> re.Pattern: - # A plain path param ({team_id}) matches a single path segment; a Starlette - # ':path' param ({key:path}) matches across '/'. Keeping plain params - # slash-bounded stops a loose regex from falsely reporting a future - # multi-segment route as already covered. - pattern = ["^"] - pos = 0 - for match in re.finditer(r"\{([^}]+)\}", route): - pattern.append(re.escape(route[pos : match.start()])) - pattern.append("[^?]+" if match.group(1).endswith(":path") else "[^/?]+") - pos = match.end() - pattern.append(re.escape(route[pos:]) + "$") - return re.compile("".join(pattern)) - - -def _test_urls() -> set: - """Every request-URL string literal across the behavior test suite. - - f-strings are reconstructed with each interpolation collapsed to a single - placeholder char, so f"/key/{target}/regenerate" becomes /key/X/regenerate. - Query strings are dropped — coverage is a path-level property. - """ - urls: set = set() - for path in sorted(TEST_DIR.glob("test_*.py")): - if path.resolve() == SELF: - continue - tree = ast.parse(path.read_text()) - for node in ast.walk(tree): - literal = None - if isinstance(node, ast.Constant) and isinstance(node.value, str): - literal = node.value - elif isinstance(node, ast.JoinedStr): - chunks = [] - for value in node.values: - if isinstance(value, ast.Constant) and isinstance(value.value, str): - chunks.append(value.value) - else: - chunks.append("X") # interpolated path / query segment - literal = "".join(chunks) - if literal and literal.startswith("/"): - urls.add(literal.split("?", 1)[0]) - return urls - - -def test_every_management_route_has_a_behavior_scenario(): - routes = _source_routes() - assert routes, "no @router routes parsed — the decorator regex is stale" - - urls = _test_urls() - uncovered = sorted( - route - for route in routes - if not any(_route_to_regex(route).match(url) for url in urls) - ) - assert ( - not uncovered - ), "management routes with no behavior-suite scenario:\n " + "\n ".join(uncovered) diff --git a/tests/proxy_behavior/management/test_scratch_teardown.py b/tests/proxy_behavior/management/test_scratch_teardown.py deleted file mode 100644 index bcb53935558..00000000000 --- a/tests/proxy_behavior/management/test_scratch_teardown.py +++ /dev/null @@ -1,61 +0,0 @@ -import pytest - -from litellm.proxy._types import LitellmUserRoles - -from .actors import ORG_A, ORG_B -from .conftest import MASTER_KEY, SCRATCH_PREFIX, create_scratch_actor - -pytestmark = pytest.mark.asyncio(loop_scope="session") - - -# The minting tests run in file order, then _b runs after their fixture -# teardown and asserts no scratch row survived in any reclaimed table. A leak -# in either direction fails _b on the next collection. - - -async def test_a_scratch_key_lands_in_db(proxy_client, prisma, scratch): - resp = await proxy_client.post( - "/key/generate", - headers={"Authorization": f"Bearer {MASTER_KEY}"}, - json={"key_alias": scratch.prefix}, - ) - assert resp.status_code == 200, resp.text - - rows = await prisma.db.litellm_verificationtoken.find_many( - where={"key_alias": scratch.prefix} - ) - assert len(rows) == 1 - - -async def test_a2_scratch_actor_lands_in_db(proxy_client, prisma, scratch): - actor = await create_scratch_actor( - prisma, - scratch.prefix, - user_role=LitellmUserRoles.ORG_ADMIN.value, - org_admin_of=(ORG_A, ORG_B), - ) - user_row = await prisma.db.litellm_usertable.find_unique( - where={"user_id": actor.user_id} - ) - assert user_row is not None - info = await proxy_client.get( - "/key/info", headers={"Authorization": f"Bearer {actor.cleartext}"} - ) - assert info.status_code == 200, info.text - memberships = await prisma.db.litellm_organizationmembership.find_many( - where={"user_id": actor.user_id} - ) - assert {m.organization_id for m in memberships} == {ORG_A, ORG_B} - - -async def test_b_scratch_namespace_is_clean(prisma): - tokens = await prisma.db.litellm_verificationtoken.find_many( - where={"key_alias": {"startswith": SCRATCH_PREFIX}} - ) - users = await prisma.db.litellm_usertable.find_many( - where={"user_id": {"startswith": SCRATCH_PREFIX}} - ) - memberships = await prisma.db.litellm_organizationmembership.find_many( - where={"user_id": {"startswith": SCRATCH_PREFIX}} - ) - assert tokens == [] and users == [] and memberships == [] diff --git a/tests/proxy_behavior/management/test_smoke.py b/tests/proxy_behavior/management/test_smoke.py deleted file mode 100644 index 4e90986ad9f..00000000000 --- a/tests/proxy_behavior/management/test_smoke.py +++ /dev/null @@ -1,28 +0,0 @@ -import pytest - -from .conftest import MASTER_KEY - -pytestmark = pytest.mark.asyncio(loop_scope="session") - - -async def test_liveliness(proxy_client): - resp = await proxy_client.get("/health/liveliness") - assert resp.status_code == 200 - - -async def test_key_generate_lands_in_db(proxy_client, prisma, scratch): - from litellm.proxy.utils import hash_token - - resp = await proxy_client.post( - "/key/generate", - headers={"Authorization": f"Bearer {MASTER_KEY}"}, - json={"key_alias": scratch.prefix}, - ) - assert resp.status_code == 200, resp.text - cleartext = resp.json()["key"] - assert cleartext.startswith("sk-") - - hashed = hash_token(cleartext) - row = await prisma.db.litellm_verificationtoken.find_unique(where={"token": hashed}) - assert row is not None - assert row.token == hashed != cleartext diff --git a/tests/proxy_behavior/management/test_team_bulk_member_add.py b/tests/proxy_behavior/management/test_team_bulk_member_add.py deleted file mode 100644 index fc83cd414e5..00000000000 --- a/tests/proxy_behavior/management/test_team_bulk_member_add.py +++ /dev/null @@ -1,105 +0,0 @@ -import pytest - -from .actors import Actor -from .conftest import create_scratch_team - -pytestmark = pytest.mark.asyncio(loop_scope="session") - - -def _member_ids(row) -> list: - return [m["user_id"] for m in (row.members_with_roles or [])] - - -async def test_team_bulk_member_add_proxy_admin_adds_explicit_members( - proxy_client, prisma, scratch, world -): - """PROXY_ADMIN bulk-adds an explicit member list to a scratch team.""" - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id) - new_member = scratch.tag("m1") - resp = await proxy_client.post( - "/team/bulk_member_add", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - json={ - "team_id": scratch.prefix, - "members": [{"user_id": new_member, "role": "user"}], - }, - ) - assert resp.status_code == 200, resp.text - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} - ) - assert row is not None and new_member in _member_ids(row) - - -async def test_team_bulk_member_add_empty_members_is_400( - proxy_client, prisma, scratch, world -): - """An empty member list (with all_users unset) is rejected 400.""" - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id) - resp = await proxy_client.post( - "/team/bulk_member_add", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - json={"team_id": scratch.prefix, "members": []}, - ) - assert resp.status_code == 400, resp.text - - -async def test_team_bulk_member_add_over_max_batch_is_400( - proxy_client, prisma, scratch, world -): - """A member list larger than the 500-member cap is rejected 400.""" - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id) - members = [ - {"user_id": f"{scratch.prefix}-u{i}", "role": "user"} for i in range(501) - ] - resp = await proxy_client.post( - "/team/bulk_member_add", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - json={"team_id": scratch.prefix, "members": members}, - ) - assert resp.status_code == 400, resp.text - - -@pytest.mark.parametrize( - "actor", - [Actor.TEAM_ADMIN, Actor.INTERNAL_USER], - ids=["team_admin", "internal_user"], -) -async def test_team_bulk_member_add_non_admin_is_401( - actor: Actor, proxy_client, prisma, scratch, world -): - """/team/bulk_member_add is neither an internal_user nor a self-managed - route — a non-proxy-admin with no org context is 401 at the route gate.""" - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id) - resp = await proxy_client.post( - "/team/bulk_member_add", - headers={"Authorization": f"Bearer {world.keys[actor].cleartext}"}, - json={ - "team_id": scratch.prefix, - "members": [{"user_id": scratch.tag("m"), "role": "user"}], - }, - ) - assert resp.status_code == 401, f"{actor.value}: {resp.status_code} {resp.text}" - - -async def test_team_bulk_member_add_all_users_proxy_admin( - proxy_client, prisma, scratch, world -): - """all_users=True pulls every user in the DB into the team. The route is - reachable only by PROXY_ADMIN (the route gate 401s every other actor — even - an org admin with organization_id in the body), so the handler's own - all_users PROXY_ADMIN gate is never the deciding check at the boundary.""" - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id) - resp = await proxy_client.post( - "/team/bulk_member_add", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - json={"team_id": scratch.prefix, "all_users": True}, - ) - assert resp.status_code == 200, resp.text - row = await prisma.db.litellm_teamtable.find_unique( - where={"team_id": scratch.prefix} - ) - assert row is not None - member_ids = _member_ids(row) - # every world actor is a user in the DB, so all are now team members - assert world.keys[Actor.INTERNAL_USER].user_id in member_ids diff --git a/tests/proxy_behavior/management/test_team_member_reset_budget.py b/tests/proxy_behavior/management/test_team_member_reset_budget.py deleted file mode 100644 index 1e55b8b6b15..00000000000 --- a/tests/proxy_behavior/management/test_team_member_reset_budget.py +++ /dev/null @@ -1,197 +0,0 @@ -import uuid - -import pytest - -from .actors import Actor -from .conftest import create_scratch_team - -pytestmark = pytest.mark.asyncio(loop_scope="session") - -_SEED_SPEND = 5.0 -_TEAM_DEFAULT_MAX_BUDGET = 100.0 -_CUSTOM_MAX_BUDGET = 50.0 - -_MATRIX = [ - ("alpha/proxy_admin", Actor.PROXY_ADMIN, "alpha", 200), - ("alpha/org_admin", Actor.ORG_ADMIN, "alpha", 200), - ("alpha/team_admin", Actor.TEAM_ADMIN, "alpha", 200), - ("alpha/internal_user", Actor.INTERNAL_USER, "alpha", 403), - ("alpha/owner", Actor.OWNER, "alpha", 403), - ("alpha/unrelated_same_org", Actor.UNRELATED_SAME_ORG, "alpha", 403), - ("alpha/cross_org_user", Actor.CROSS_ORG_USER, "alpha", 403), - ("alpha/service_account", Actor.SERVICE_ACCOUNT, "alpha", 403), - ("alpha/org_b_admin", Actor.ORG_B_ADMIN, "alpha", 403), - ("beta/proxy_admin", Actor.PROXY_ADMIN, "beta", 200), - ("beta/org_admin", Actor.ORG_ADMIN, "beta", 403), - ("beta/team_admin", Actor.TEAM_ADMIN, "beta", 403), - ("beta/org_b_admin", Actor.ORG_B_ADMIN, "beta", 200), -] - - -async def _seed_budget(prisma, budget_id: str, max_budget: float) -> str: - await prisma.db.litellm_budgettable.create( - data={ - "budget_id": budget_id, - "max_budget": max_budget, - "created_by": "phase4-scratch", - "updated_by": "phase4-scratch", - } - ) - return budget_id - - -async def _seed_team_with_default_budget(prisma, world, shape: str, team_id: str, scratch) -> str: - default_budget_id = await _seed_budget(prisma, scratch.tag("team-default-budget"), _TEAM_DEFAULT_MAX_BUDGET) - metadata = {"team_member_budget_id": default_budget_id} - if shape == "alpha": - await create_scratch_team( - prisma, - team_id, - organization_id=world.org_a_id, - admin_user_ids=[world.keys[Actor.TEAM_ADMIN].user_id], - metadata=metadata, - ) - elif shape == "beta": - await create_scratch_team(prisma, team_id, organization_id=world.org_b_id, metadata=metadata) - else: # pragma: no cover - guard - pytest.fail(f"unknown shape={shape}") - return default_budget_id - - -async def _seed_custom_member(prisma, team_id: str, member_id: str, scratch) -> str: - custom_budget_id = await _seed_budget(prisma, scratch.tag("custom-budget"), _CUSTOM_MAX_BUDGET) - await prisma.db.litellm_teammembership.create( - data={ - "user_id": member_id, - "team_id": team_id, - "spend": _SEED_SPEND, - "litellm_budget_table": {"connect": {"budget_id": custom_budget_id}}, - } - ) - return custom_budget_id - - -async def _membership(prisma, team_id: str, member_id: str): - row = await prisma.db.litellm_teammembership.find_unique( - where={"user_id_team_id": {"user_id": member_id, "team_id": team_id}} - ) - assert row is not None - return row - - -@pytest.mark.parametrize( - "actor,shape,expected_status", - [(a, sh, s) for (_id, a, sh, s) in _MATRIX], - ids=[s[0] for s in _MATRIX], -) -async def test_team_member_reset_budget_authz_matrix( - actor: Actor, - shape: str, - expected_status: int, - proxy_client, - prisma, - scratch, - world, -): - member_id = scratch.tag("member") - default_budget_id = await _seed_team_with_default_budget(prisma, world, shape, scratch.prefix, scratch) - custom_budget_id = await _seed_custom_member(prisma, scratch.prefix, member_id, scratch) - caller = world.keys[actor] - - resp = await proxy_client.post( - f"/team/{scratch.prefix}/member/{member_id}/reset_budget", - headers={"Authorization": f"Bearer {caller.cleartext}"}, - ) - assert resp.status_code == expected_status, f"{actor.value} {shape}: {resp.status_code} {resp.text}" - - row = await _membership(prisma, scratch.prefix, member_id) - assert row.spend == _SEED_SPEND, "reset_budget must never touch spend" - if expected_status == 200: - assert row.budget_id == default_budget_id - body = resp.json() - assert body["budget_id"] == default_budget_id - assert body["previous_budget_id"] == custom_budget_id - assert body["budget_source"] == "team_default" - else: - assert row.budget_id == custom_budget_id, "denied but budget relinked" - - -async def test_team_member_reset_budget_leaves_shared_default_row_untouched(proxy_client, prisma, scratch, world): - member_id = scratch.tag("member") - default_budget_id = await _seed_team_with_default_budget(prisma, world, "alpha", scratch.prefix, scratch) - await _seed_custom_member(prisma, scratch.prefix, member_id, scratch) - - resp = await proxy_client.post( - f"/team/{scratch.prefix}/member/{member_id}/reset_budget", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - ) - assert resp.status_code == 200, resp.text - - default_row = await prisma.db.litellm_budgettable.find_unique(where={"budget_id": default_budget_id}) - assert default_row is not None and default_row.max_budget == _TEAM_DEFAULT_MAX_BUDGET - - info = await proxy_client.get( - f"/team/info?team_id={scratch.prefix}", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - ) - assert info.status_code == 200, info.text - memberships = {tm["user_id"]: tm for tm in info.json()["team_memberships"]} - assert memberships[member_id]["budget_source"] == "team_default" - assert memberships[member_id]["litellm_budget_table"]["max_budget"] == _TEAM_DEFAULT_MAX_BUDGET - - -async def test_team_member_reset_budget_without_team_default_detaches_member(proxy_client, prisma, scratch, world): - member_id = scratch.tag("member") - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id) - await _seed_custom_member(prisma, scratch.prefix, member_id, scratch) - - resp = await proxy_client.post( - f"/team/{scratch.prefix}/member/{member_id}/reset_budget", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - ) - assert resp.status_code == 200, resp.text - assert resp.json()["budget_id"] is None - assert resp.json()["budget_source"] == "none" - - row = await _membership(prisma, scratch.prefix, member_id) - assert row.budget_id is None - assert row.spend == _SEED_SPEND - - -async def test_team_member_reset_budget_with_deleted_team_default_detaches_member(proxy_client, prisma, scratch, world): - member_id = scratch.tag("member") - await create_scratch_team( - prisma, - scratch.prefix, - organization_id=world.org_a_id, - metadata={"team_member_budget_id": scratch.tag("deleted-budget")}, - ) - await _seed_custom_member(prisma, scratch.prefix, member_id, scratch) - - resp = await proxy_client.post( - f"/team/{scratch.prefix}/member/{member_id}/reset_budget", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - ) - assert resp.status_code == 200, resp.text - assert resp.json()["budget_id"] is None - assert resp.json()["budget_source"] == "none" - - row = await _membership(prisma, scratch.prefix, member_id) - assert row.budget_id is None - - -async def test_team_member_reset_budget_missing_team_is_404(proxy_client, world): - resp = await proxy_client.post( - f"/team/behavior-pin-no-such-team/member/{uuid.uuid4().hex}/reset_budget", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - ) - assert resp.status_code == 404, resp.text - - -async def test_team_member_reset_budget_missing_membership_is_404(proxy_client, prisma, scratch, world): - await create_scratch_team(prisma, scratch.prefix, organization_id=world.org_a_id) - resp = await proxy_client.post( - f"/team/{scratch.prefix}/member/{uuid.uuid4().hex}/reset_budget", - headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, - ) - assert resp.status_code == 404, resp.text diff --git a/tests/proxy_behavior/management/test_world_seed.py b/tests/proxy_behavior/management/test_world_seed.py deleted file mode 100644 index 00f9540c9c3..00000000000 --- a/tests/proxy_behavior/management/test_world_seed.py +++ /dev/null @@ -1,30 +0,0 @@ -import pytest - -from .actors import Actor - -pytestmark = pytest.mark.asyncio(loop_scope="session") - - -@pytest.mark.parametrize("actor", list(Actor), ids=[a.value for a in Actor]) -async def test_each_actor_can_self_info(actor, proxy_client, world): - seeded = world.keys[actor] - resp = await proxy_client.get( - "/key/info", - headers={"Authorization": f"Bearer {seeded.cleartext}"}, - ) - assert resp.status_code == 200, f"{actor.value}: {resp.text}" - body = resp.json() - assert body.get("key") == seeded.hashed - assert body["info"].get("user_id") == seeded.user_id - - -async def test_proxy_admin_actor_can_create_keys_for_others(proxy_client, world): - seeder = world.keys[Actor.PROXY_ADMIN] - target_user_id = world.keys[Actor.OWNER].user_id - - resp = await proxy_client.post( - "/key/generate", - headers={"Authorization": f"Bearer {seeder.cleartext}"}, - json={"key_alias": "smoke-proxy-admin-bypass", "user_id": target_user_id}, - ) - assert resp.status_code == 200, resp.text diff --git a/tests/proxy_behavior/spend/conftest.py b/tests/proxy_behavior/spend/conftest.py deleted file mode 100644 index 0b918401eac..00000000000 --- a/tests/proxy_behavior/spend/conftest.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Session-scoped Prisma client for spend-rollup behavior tests against a real Postgres.""" - -import pytest_asyncio -from prisma import Prisma - - -@pytest_asyncio.fixture(scope="session", loop_scope="session") -async def db(): - client = Prisma() - await client.connect() - yield client - await client.disconnect() diff --git a/tests/proxy_unit_tests/azure_fine_tune.jsonl b/tests/proxy_unit_tests/azure_fine_tune.jsonl deleted file mode 100644 index ef41bd97731..00000000000 --- a/tests/proxy_unit_tests/azure_fine_tune.jsonl +++ /dev/null @@ -1,12 +0,0 @@ -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How far is the Moon from Earth?"}, {"role": "assistant", "content": "Around 384,400 kilometers. Give or take a few, like that really matters."}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How far is the Moon from Earth?"}, {"role": "assistant", "content": "Around 384,400 kilometers. Give or take a few, like that really matters."}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How far is the Moon from Earth?"}, {"role": "assistant", "content": "Around 384,400 kilometers. Give or take a few, like that really matters."}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]} -{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How far is the Moon from Earth?"}, {"role": "assistant", "content": "Around 384,400 kilometers. Give or take a few, like that really matters."}]} \ No newline at end of file diff --git a/tests/proxy_unit_tests/batch_job_results_furniture.jsonl b/tests/proxy_unit_tests/batch_job_results_furniture.jsonl deleted file mode 100644 index 05448952a0f..00000000000 --- a/tests/proxy_unit_tests/batch_job_results_furniture.jsonl +++ /dev/null @@ -1,2 +0,0 @@ -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} \ No newline at end of file diff --git a/tests/proxy_unit_tests/conftest.py b/tests/proxy_unit_tests/conftest.py deleted file mode 100644 index 148751c33f2..00000000000 --- a/tests/proxy_unit_tests/conftest.py +++ /dev/null @@ -1,150 +0,0 @@ -# conftest.py - -import asyncio -import copy -import inspect -import warnings - -import pytest - - -import litellm -import litellm.proxy.proxy_server - - -# Top-level assignments of these types are the ones importlib.reload(litellm) -# would have effectively reset. We snapshot them at conftest import time and -# deep-copy the snapshot back before every test. -_SNAPSHOT_TYPES = (list, dict, set, tuple, str, int, float, bool, bytes) - - -def _snapshot_mutable_state(module): - """Capture a per-module snapshot of primitive and collection attributes.""" - snapshot = {} - for attr in list(vars(module)): - if attr.startswith("_"): - continue - try: - value = getattr(module, attr) - except Exception as exc: - warnings.warn( - f"conftest: could not read {module.__name__}.{attr} during snapshot: {exc}", - stacklevel=2, - ) - continue - if value is None or isinstance(value, _SNAPSHOT_TYPES): - try: - snapshot[attr] = copy.deepcopy(value) - except Exception as exc: - warnings.warn( - f"conftest: could not snapshot {module.__name__}.{attr}: {exc}", - stacklevel=2, - ) - return snapshot - - -def _restore_mutable_state(module, snapshot): - for attr, default in snapshot.items(): - try: - setattr(module, attr, copy.deepcopy(default)) - except Exception as exc: - warnings.warn( - f"conftest: could not restore {module.__name__}.{attr}: {exc}", - stacklevel=2, - ) - - -def _collect_flushable_caches(): - """Return (module, attr) pairs whose values expose flush_cache().""" - targets = [] - for module in (litellm, litellm.proxy.proxy_server): - for attr in list(vars(module)): - if attr.startswith("_"): - continue - try: - value = getattr(module, attr) - except Exception: - continue - # Only instances — a class reference has an unbound flush_cache - # that can't be called without a self argument. - if inspect.isclass(value) or inspect.ismodule(value): - continue - if callable(getattr(value, "flush_cache", None)): - targets.append((module, attr)) - return targets - - -def _flush_caches(targets): - for module, attr in targets: - try: - value = getattr(module, attr) - except Exception: - continue - flush = getattr(value, "flush_cache", None) - if callable(flush): - try: - flush() - except Exception as exc: - warnings.warn( - f"conftest: flush_cache failed on {module.__name__}.{attr}: {exc}", - stacklevel=2, - ) - - -# Snapshot once at conftest import — these are the "clean" module states. -_LITELLM_STATE = _snapshot_mutable_state(litellm) -_PROXY_SERVER_STATE = _snapshot_mutable_state(litellm.proxy.proxy_server) -_FLUSHABLE_CACHES = _collect_flushable_caches() - - -@pytest.fixture(scope="function", autouse=True) -def setup_and_teardown(): - """Reset mutable module state on litellm and proxy_server before each test. - - Replaces a previous importlib.reload(litellm) approach that cost ~17s - per test (re-executing the full litellm __init__ import chain). - - What IS reset: - - Top-level module attributes of type list / dict / set / tuple - / str / int / float / bool / bytes, and None-valued attributes. - These cover callback lists, general_settings, master_key, - premium_user, prisma_client, etc. — anything the old reload() reset - by re-executing the module body. - - Any module-level object instance that exposes flush_cache() (the - DualCache and LLMClientCache family), which handles cache state - that can't round-trip through deepcopy because of internal locks. - - What is NOT reset: - - Class instances without flush_cache() (e.g. ProxyLogging, - JWTHandler, FastAPI routers, loggers). If a test mutates such an - instance in-place (setattr on the instance, appending to one of - its internal lists, etc.), the mutation will leak into later tests. - Use pytest's monkeypatch.setattr() or a local fixture for those - cases — don't rely on this autouse fixture to undo them. - """ - _restore_mutable_state(litellm, _LITELLM_STATE) - _restore_mutable_state(litellm.proxy.proxy_server, _PROXY_SERVER_STATE) - _flush_caches(_FLUSHABLE_CACHES) - - loop = asyncio.get_event_loop_policy().new_event_loop() - asyncio.set_event_loop(loop) - try: - yield - finally: - loop.close() - asyncio.set_event_loop(None) - - -def pytest_collection_modifyitems(config, items): - # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests - custom_logger_tests = [ - item for item in items if "custom_logger" in item.parent.name - ] - other_tests = [item for item in items if "custom_logger" not in item.parent.name] - - # Sort tests based on their names - custom_logger_tests.sort(key=lambda x: x.name) - other_tests.sort(key=lambda x: x.name) - - # Reorder the items list - items[:] = custom_logger_tests + other_tests diff --git a/tests/proxy_unit_tests/data_map.txt b/tests/proxy_unit_tests/data_map.txt deleted file mode 100644 index e8077595f47..00000000000 Binary files a/tests/proxy_unit_tests/data_map.txt and /dev/null differ diff --git a/tests/proxy_unit_tests/eagle.wav b/tests/proxy_unit_tests/eagle.wav deleted file mode 100644 index 1c23657859e..00000000000 Binary files a/tests/proxy_unit_tests/eagle.wav and /dev/null differ diff --git a/tests/proxy_unit_tests/gettysburg.wav b/tests/proxy_unit_tests/gettysburg.wav deleted file mode 100644 index 9690f521e84..00000000000 Binary files a/tests/proxy_unit_tests/gettysburg.wav and /dev/null differ diff --git a/tests/proxy_unit_tests/large_text.py b/tests/proxy_unit_tests/large_text.py deleted file mode 100644 index 86904a6d148..00000000000 --- a/tests/proxy_unit_tests/large_text.py +++ /dev/null @@ -1,112 +0,0 @@ -text = """ -Alexander the Great -This article is about the ancient king of Macedonia. For other uses, see Alexander the Great (disambiguation). -Alexander III of Macedon (Ancient Greek: Ἀλέξανδρος, romanized: Alexandros; 20/21 July 356 BC – 10/11 June 323 BC), most commonly known as Alexander the Great,[c] was a king of the ancient Greek kingdom of Macedon.[d] He succeeded his father Philip II to the throne in 336 BC at the age of 20 and spent most of his ruling years conducting a lengthy military campaign throughout Western Asia, Central Asia, parts of South Asia, and Egypt. By the age of 30, he had created one of the largest empires in history, stretching from Greece to northwestern India.[1] He was undefeated in battle and is widely considered to be one of history's greatest and most successful military commanders.[2][3] - -Until the age of 16, Alexander was tutored by Aristotle. In 335 BC, shortly after his assumption of kingship over Macedon, he campaigned in the Balkans and reasserted control over Thrace and parts of Illyria before marching on the city of Thebes, which was subsequently destroyed in battle. Alexander then led the League of Corinth, and used his authority to launch the pan-Hellenic project envisaged by his father, assuming leadership over all Greeks in their conquest of Persia.[4][5] - -In 334 BC, he invaded the Achaemenid Persian Empire and began a series of campaigns that lasted for 10 years. Following his conquest of Asia Minor, Alexander broke the power of Achaemenid Persia in a series of decisive battles, including those at Issus and Gaugamela; he subsequently overthrew Darius III and conquered the Achaemenid Empire in its entirety.[e] After the fall of Persia, the Macedonian Empire held a vast swath of territory between the Adriatic Sea and the Indus River. Alexander endeavored to reach the "ends of the world and the Great Outer Sea" and invaded India in 326 BC, achieving an important victory over Porus, an ancient Indian king of present-day Punjab, at the Battle of the Hydaspes. Due to the demand of his homesick troops, he eventually turned back at the Beas River and later died in 323 BC in Babylon, the city of Mesopotamia that he had planned to establish as his empire's capital. Alexander's death left unexecuted an additional series of planned military and mercantile campaigns that would have begun with a Greek invasion of Arabia. In the years following his death, a series of civil wars broke out across the Macedonian Empire, eventually leading to its disintegration at the hands of the Diadochi. - -With his death marking the start of the Hellenistic period, Alexander's legacy includes the cultural diffusion and syncretism that his conquests engendered, such as Greco-Buddhism and Hellenistic Judaism. He founded more than twenty cities, with the most prominent being the city of Alexandria in Egypt. Alexander's settlement of Greek colonists and the resulting spread of Greek culture led to the overwhelming dominance of Hellenistic civilization and influence as far east as the Indian subcontinent. The Hellenistic period developed through the Roman Empire into modern Western culture; the Greek language became the lingua franca of the region and was the predominant language of the Byzantine Empire up until its collapse in the mid-15th century AD. Alexander became legendary as a classical hero in the mould of Achilles, featuring prominently in the historical and mythical traditions of both Greek and non-Greek cultures. His military achievements and unprecedented enduring successes in battle made him the measure against which many later military leaders would compare themselves,[f] and his tactics remain a significant subject of study in military academies worldwide.[6] Legends of Alexander's exploits coalesced into the third-century Alexander Romance which, in the premodern period, went through over one hundred recensions, translations, and derivations and was translated into almost every European vernacular and every language of the Islamic world.[7] After the Bible, it was the most popular form of European literature.[8] - -Early life - -Lineage and childhood - -Alexander III was born in Pella, the capital of the Kingdom of Macedon,[9] on the sixth day of the ancient Greek month of Hekatombaion, which probably corresponds to 20 July 356 BC (although the exact date is uncertain).[10][11] He was the son of the erstwhile king of Macedon, Philip II, and his fourth wife, Olympias (daughter of Neoptolemus I, king of Epirus).[12][g] Although Philip had seven or eight wives, Olympias was his principal wife for some time, likely because she gave birth to Alexander.[13] - -Several legends surround Alexander's birth and childhood.[14] According to the ancient Greek biographer Plutarch, on the eve of the consummation of her marriage to Philip, Olympias dreamed that her womb was struck by a thunderbolt that caused a flame to spread "far and wide" before dying away. Sometime after the wedding, Philip is said to have seen himself, in a dream, securing his wife's womb with a seal engraved with a lion's image.[15] Plutarch offered a variety of interpretations for these dreams: that Olympias was pregnant before her marriage, indicated by the sealing of her womb; or that Alexander's father was Zeus. Ancient commentators were divided about whether the ambitious Olympias promulgated the story of Alexander's divine parentage, variously claiming that she had told Alexander, or that she dismissed the suggestion as impious.[15] - -On the day Alexander was born, Philip was preparing a siege on the city of Potidea on the peninsula of Chalcidice. That same day, Philip received news that his general Parmenion had defeated the combined Illyrian and Paeonian armies and that his horses had won at the Olympic Games. It was also said that on this day, the Temple of Artemis in Ephesus, one of the Seven Wonders of the World, burnt down. This led Hegesias of Magnesia to say that it had burnt down because Artemis was away, attending the birth of Alexander.[16] Such legends may have emerged when Alexander was king, and possibly at his instigation, to show that he was superhuman and destined for greatness from conception.[14] - -In his early years, Alexander was raised by a nurse, Lanike, sister of Alexander's future general Cleitus the Black. Later in his childhood, Alexander was tutored by the strict Leonidas, a relative of his mother, and by Lysimachus of Acarnania.[17] Alexander was raised in the manner of noble Macedonian youths, learning to read, play the lyre, ride, fight, and hunt.[18] When Alexander was ten years old, a trader from Thessaly brought Philip a horse, which he offered to sell for thirteen talents. The horse refused to be mounted, and Philip ordered it away. Alexander, however, detecting the horse's fear of its own shadow, asked to tame the horse, which he eventually managed.[14] Plutarch stated that Philip, overjoyed at this display of courage and ambition, kissed his son tearfully, declaring: "My boy, you must find a kingdom big enough for your ambitions. Macedon is too small for you", and bought the horse for him.[19] Alexander named it Bucephalas, meaning "ox-head". Bucephalas carried Alexander as far as India. When the animal died (because of old age, according to Plutarch, at age 30), Alexander named a city after him, Bucephala.[20] - -Education - -When Alexander was 13, Philip began to search for a tutor, and considered such academics as Isocrates and Speusippus, the latter offering to resign from his stewardship of the Academy to take up the post. In the end, Philip chose Aristotle and provided the Temple of the Nymphs at Mieza as a classroom. In return for teaching Alexander, Philip agreed to rebuild Aristotle's hometown of Stageira, which Philip had razed, and to repopulate it by buying and freeing the ex-citizens who were slaves, or pardoning those who were in exile.[21] - -Mieza was like a boarding school for Alexander and the children of Macedonian nobles, such as Ptolemy, Hephaistion, and Cassander. Many of these students would become his friends and future generals, and are often known as the "Companions". Aristotle taught Alexander and his companions about medicine, philosophy, morals, religion, logic, and art. Under Aristotle's tutelage, Alexander developed a passion for the works of Homer, and in particular the Iliad; Aristotle gave him an annotated copy, which Alexander later carried on his campaigns.[22] Alexander was able to quote Euripides from memory.[23] - -During his youth, Alexander was also acquainted with Persian exiles at the Macedonian court, who received the protection of Philip II for several years as they opposed Artaxerxes III.[24][25][26] Among them were Artabazos II and his daughter Barsine, possible future mistress of Alexander, who resided at the Macedonian court from 352 to 342 BC, as well as Amminapes, future satrap of Alexander, or a Persian nobleman named Sisines.[24][27][28][29] This gave the Macedonian court a good knowledge of Persian issues, and may even have influenced some of the innovations in the management of the Macedonian state.[27] - -Suda writes that Anaximenes of Lampsacus was one of Alexander's teachers, and that Anaximenes also accompanied Alexander on his campaigns.[30] - -Heir of Philip II - -Regency and ascent of Macedon - -Main articles: Philip II of Macedon and Rise of Macedon -Further information: History of Macedonia (ancient kingdom) -At the age of 16, Alexander's education under Aristotle ended. Philip II had waged war against the Thracians to the north, which left Alexander in charge as regent and heir apparent.[14] During Philip's absence, the Thracian tribe of Maedi revolted against Macedonia. Alexander responded quickly and drove them from their territory. The territory was colonized, and a city, named Alexandropolis, was founded.[31] - -Upon Philip's return, Alexander was dispatched with a small force to subdue the revolts in southern Thrace. Campaigning against the Greek city of Perinthus, Alexander reportedly saved his father's life. Meanwhile, the city of Amphissa began to work lands that were sacred to Apollo near Delphi, a sacrilege that gave Philip the opportunity to further intervene in Greek affairs. While Philip was occupied in Thrace, Alexander was ordered to muster an army for a campaign in southern Greece. Concerned that other Greek states might intervene, Alexander made it look as though he was preparing to attack Illyria instead. During this turmoil, the Illyrians invaded Macedonia, only to be repelled by Alexander.[32] - -Philip and his army joined his son in 338 BC, and they marched south through Thermopylae, taking it after stubborn resistance from its Theban garrison. They went on to occupy the city of Elatea, only a few days' march from both Athens and Thebes. The Athenians, led by Demosthenes, voted to seek alliance with Thebes against Macedonia. Both Athens and Philip sent embassies to win Thebes's favour, but Athens won the contest.[33] Philip marched on Amphissa (ostensibly acting on the request of the Amphictyonic League), capturing the mercenaries sent there by Demosthenes and accepting the city's surrender. Philip then returned to Elatea, sending a final offer of peace to Athens and Thebes, who both rejected it.[34] - -As Philip marched south, his opponents blocked him near Chaeronea, Boeotia. During the ensuing Battle of Chaeronea, Philip commanded the right wing and Alexander the left, accompanied by a group of Philip's trusted generals. According to the ancient sources, the two sides fought bitterly for some time. Philip deliberately commanded his troops to retreat, counting on the untested Athenian hoplites to follow, thus breaking their line. Alexander was the first to break the Theban lines, followed by Philip's generals. Having damaged the enemy's cohesion, Philip ordered his troops to press forward and quickly routed them. With the Athenians lost, the Thebans were surrounded. Left to fight alone, they were defeated.[35] - -After the victory at Chaeronea, Philip and Alexander marched unopposed into the Peloponnese, welcomed by all cities; however, when they reached Sparta, they were refused, but did not resort to war.[36] At Corinth, Philip established a "Hellenic Alliance" (modelled on the old anti-Persian alliance of the Greco-Persian Wars), which included most Greek city-states except Sparta. Philip was then named Hegemon (often translated as "Supreme Commander") of this league (known by modern scholars as the League of Corinth), and announced his plans to attack the Persian Empire.[37][38] - -Exile and return - -When Philip returned to Pella, he fell in love with and married Cleopatra Eurydice in 338 BC,[39] the niece of his general Attalus.[40] The marriage made Alexander's position as heir less secure, since any son of Cleopatra Eurydice would be a fully Macedonian heir, while Alexander was only half-Macedonian.[41] During the wedding banquet, a drunken Attalus publicly prayed to the gods that the union would produce a legitimate heir.[40] - -At the wedding of Cleopatra, whom Philip fell in love with and married, she being much too young for him, her uncle Attalus in his drink desired the Macedonians would implore the gods to give them a lawful successor to the kingdom by his niece. This so irritated Alexander, that throwing one of the cups at his head, "You villain," said he, "what, am I then a bastard?" Then Philip, taking Attalus's part, rose up and would have run his son through; but by good fortune for them both, either his over-hasty rage, or the wine he had drunk, made his foot slip, so that he fell down on the floor. At which Alexander reproachfully insulted over him: "See there," said he, "the man who makes preparations to pass out of Europe into Asia, overturned in passing from one seat to another." - -— Plutarch, describing the feud at Philip's wedding.[42]none -In 337 BC, Alexander fled Macedon with his mother, dropping her off with her brother, King Alexander I of Epirus in Dodona, capital of the Molossians.[43] He continued to Illyria,[43] where he sought refuge with one or more Illyrian kings, perhaps with Glaucias, and was treated as a guest, despite having defeated them in battle a few years before.[44] However, it appears Philip never intended to disown his politically and militarily trained son.[43] Accordingly, Alexander returned to Macedon after six months due to the efforts of a family friend, Demaratus, who mediated between the two parties.[45] - -In the following year, the Persian satrap (governor) of Caria, Pixodarus, offered his eldest daughter to Alexander's half-brother, Philip Arrhidaeus.[43] Olympias and several of Alexander's friends suggested this showed Philip intended to make Arrhidaeus his heir.[43] Alexander reacted by sending an actor, Thessalus of Corinth, to tell Pixodarus that he should not offer his daughter's hand to an illegitimate son, but instead to Alexander. When Philip heard of this, he stopped the negotiations and scolded Alexander for wishing to marry the daughter of a Carian, explaining that he wanted a better bride for him.[43] Philip exiled four of Alexander's friends, Harpalus, Nearchus, Ptolemy and Erigyius, and had the Corinthians bring Thessalus to him in chains.[46] - -King of Macedon - -Accession - -Further information: Government of Macedonia (ancient kingdom) -In summer 336 BC, while at Aegae attending the wedding of his daughter Cleopatra to Olympias's brother, Alexander I of Epirus, Philip was assassinated by the captain of his bodyguards, Pausanias.[h] As Pausanias tried to escape, he tripped over a vine and was killed by his pursuers, including two of Alexander's companions, Perdiccas and Leonnatus. Alexander was proclaimed king on the spot by the nobles and army at the age of 20.[47][48][49] - -Consolidation of power - -Alexander began his reign by eliminating potential rivals to the throne. He had his cousin, the former Amyntas IV, executed.[51] He also had two Macedonian princes from the region of Lyncestis killed for having been involved in his father's assassination, but spared a third, Alexander Lyncestes. Olympias had Cleopatra Eurydice, and Europa, her daughter by Philip, burned alive. When Alexander learned about this, he was furious. Alexander also ordered the murder of Attalus,[51] who was in command of the advance guard of the army in Asia Minor and Cleopatra's uncle.[52] - -Attalus was at that time corresponding with Demosthenes, regarding the possibility of defecting to Athens. Attalus also had severely insulted Alexander, and following Cleopatra's murder, Alexander may have considered him too dangerous to be left alive.[52] Alexander spared Arrhidaeus, who was by all accounts mentally disabled, possibly as a result of poisoning by Olympias.[47][49][53] - -News of Philip's death roused many states into revolt, including Thebes, Athens, Thessaly, and the Thracian tribes north of Macedon. When news of the revolts reached Alexander, he responded quickly. Though advised to use diplomacy, Alexander mustered 3,000 Macedonian cavalry and rode south towards Thessaly. He found the Thessalian army occupying the pass between Mount Olympus and Mount Ossa, and ordered his men to ride over Mount Ossa. When the Thessalians awoke the next day, they found Alexander in their rear and promptly surrendered, adding their cavalry to Alexander's force. He then continued south towards the Peloponnese.[54] - -Alexander stopped at Thermopylae, where he was recognized as the leader of the Amphictyonic League before heading south to Corinth. Athens sued for peace and Alexander pardoned the rebels. The famous encounter between Alexander and Diogenes the Cynic occurred during Alexander's stay in Corinth. When Alexander asked Diogenes what he could do for him, the philosopher disdainfully asked Alexander to stand a little to the side, as he was blocking the sunlight.[55] This reply apparently delighted Alexander, who is reported to have said "But verily, if I were not Alexander, I would like to be Diogenes."[56] At Corinth, Alexander took the title of Hegemon ("leader") and, like Philip, was appointed commander for the coming war against Persia. He also received news of a Thracian uprising.[57] - -Balkan campaign - -Main article: Alexander's Balkan campaign -Before crossing to Asia, Alexander wanted to safeguard his northern borders. In the spring of 335 BC, he advanced to suppress several revolts. Starting from Amphipolis, he travelled east into the country of the "Independent Thracians"; and at Mount Haemus, the Macedonian army attacked and defeated the Thracian forces manning the heights.[58] The Macedonians marched into the country of the Triballi, and defeated their army near the Lyginus river[59] (a tributary of the Danube). Alexander then marched for three days to the Danube, encountering the Getae tribe on the opposite shore. Crossing the river at night, he surprised them and forced their army to retreat after the first cavalry skirmish.[60] - -News then reached Alexander that the Illyrian chieftain Cleitus and King Glaukias of the Taulantii were in open revolt against his authority. Marching west into Illyria, Alexander defeated each in turn, forcing the two rulers to flee with their troops. With these victories, he secured his northern frontier.[61] - -Destruction of Thebes - -While Alexander campaigned north, the Thebans and Athenians rebelled once again. Alexander immediately headed south.[62] While the other cities again hesitated, Thebes decided to fight. The Theban resistance was ineffective, and Alexander razed the city and divided its territory between the other Boeotian cities. The end of Thebes cowed Athens, leaving all of Greece temporarily at peace.[62] Alexander then set out on his Asian campaign, leaving Antipater as regent.[63] - -Conquest of the Achaemenid Persian Empire - -Main articles: Wars of Alexander the Great and Chronology of the expedition of Alexander the Great into Asia -Asia Minor - -Further information: Battle of the Granicus, Siege of Halicarnassus, and Siege of Miletus -After his victory at the Battle of Chaeronea (338 BC), Philip II began the work of establishing himself as hēgemṓn (Greek: ἡγεμών) of a league which according to Diodorus was to wage a campaign against the Persians for the sundry grievances Greece suffered in 480 and free the Greek cities of the western coast and islands from Achaemenid rule. In 336 he sent Parmenion, Amyntas, Andromenes, Attalus, and an army of 10,000 men into Anatolia to make preparations for an invasion.[64][65] At first, all went well. The Greek cities on the western coast of Anatolia revolted until the news arrived that Philip had been murdered and had been succeeded by his young son Alexander. The Macedonians were demoralized by Philip's death and were subsequently defeated near Magnesia by the Achaemenids under the command of the mercenary Memnon of Rhodes.[64][65] - -Taking over the invasion project of Philip II, Alexander's army crossed the Hellespont in 334 BC with approximately 48,100 soldiers, 6,100 cavalry and a fleet of 120 ships with crews numbering 38,000,[62] drawn from Macedon and various Greek city-states, mercenaries, and feudally raised soldiers from Thrace, Paionia, and Illyria.[66][i] He showed his intent to conquer the entirety of the Persian Empire by throwing a spear into Asian soil and saying he accepted Asia as a gift from the gods. This also showed Alexander's eagerness to fight, in contrast to his father's preference for diplomacy.[62] - -After an initial victory against Persian forces at the Battle of the Granicus, Alexander accepted the surrender of the Persian provincial capital and treasury of Sardis; he then proceeded along the Ionian coast, granting autonomy and democracy to the cities. Miletus, held by Achaemenid forces, required a delicate siege operation, with Persian naval forces nearby. Further south, at Halicarnassus, in Caria, Alexander successfully waged his first large-scale siege, eventually forcing his opponents, the mercenary captain Memnon of Rhodes and the Persian satrap of Caria, Orontobates, to withdraw by sea.[67] Alexander left the government of Caria to a member of the Hecatomnid dynasty, Ada, who adopted Alexander.[68] - -From Halicarnassus, Alexander proceeded into mountainous Lycia and the Pamphylian plain, asserting control over all coastal cities to deny the Persians naval bases. From Pamphylia onwards the coast held no major ports and Alexander moved inland. At Termessos, Alexander humbled but did not storm the Pisidian city.[69] At the ancient Phrygian capital of Gordium, Alexander "undid" the hitherto unsolvable Gordian Knot, a feat said to await the future "king of Asia".[70] According to the story, Alexander proclaimed that it did not matter how the knot was undone and hacked it apart with his sword.[71] - -The Levant and Syria - -Further information: Battle of Issus and Siege of Tyre (332 BC) -In spring 333 BC, Alexander crossed the Taurus into Cilicia. After a long pause due to an illness, he marched on towards Syria. Though outmanoeuvered by Darius's significantly larger army, he marched back to Cilicia, where he defeated Darius at Issus. Darius fled the battle, causing his army to collapse, and left behind his wife, his two daughters, his mother Sisygambis, and a fabulous treasure.[72] He offered a peace treaty that included the lands he had already lost, and a ransom of 10,000 talents for his family. Alexander replied that since he was now king of Asia, it was he alone who decided territorial divisions.[73] Alexander proceeded to take possession of Syria, and most of the coast of the Levant.[68] In the following year, 332 BC, he was forced to attack Tyre, which he captured after a long and difficult siege.[74][75] The men of military age were massacred and the women and children sold into slavery.[76] - -Egypt - -Further information: Siege of Gaza (332 BCE) -When Alexander destroyed Tyre, most of the towns on the route to Egypt quickly capitulated. However, Alexander was met with resistance at Gaza. The stronghold was heavily fortified and built on a hill, requiring a siege. When "his engineers pointed out to him that because of the height of the mound it would be impossible... this encouraged Alexander all the more to make the attempt".[77] After three unsuccessful assaults, the stronghold fell, but not before Alexander had received a serious shoulder wound. As in Tyre, men of military age were put to the sword and the women and children were sold into slavery.[78] -""" diff --git a/tests/proxy_unit_tests/messages_with_counts.py b/tests/proxy_unit_tests/messages_with_counts.py deleted file mode 100644 index da27a975520..00000000000 --- a/tests/proxy_unit_tests/messages_with_counts.py +++ /dev/null @@ -1,733 +0,0 @@ -system_message_short = { - "message": { - "role": "system", - "content": "You are a bot.", - }, - "count": 12, -} - -system_message = { - "message": { - "role": "system", - "content": "You are a helpful, pattern-following assistant that translates corporate jargon into plain English.", - }, - "count": 25, -} - -system_message_long = { - "message": { - "role": "system", - "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.", - }, - "count": 31, -} - -system_message_unicode = { - "message": { - "role": "system", - "content": "á", - }, - "count": 8, -} - -system_message_with_name = { - "message": { - "role": "system", - "name": "example_user", - "content": "New synergies will help drive top-line growth.", - }, - "count": 20, -} - -user_message = { - "message": { - "role": "user", - "content": "Hello, how are you?", - }, - "count": 13, -} - -user_message_unicode = { - "message": { - "role": "user", - "content": "á", - }, - "count": 8, -} - -user_message_perf = { - "message": { - "role": "user", - "content": "What happens in a performance review?", - }, - "count": 14, -} - -assistant_message_perf = { - "message": { - "role": "assistant", - "content": "During the performance review at Contoso Electronics, the supervisor will discuss the employee's performance over the past year and provide feedback on areas for improvement. They will also provide an opportunity for the employee to discuss their goals and objectives for the upcoming year. The review is a two-way dialogue between managers and employees, and employees will receive a written summary of their performance review which will include a rating of their performance, feedback, and goals and objectives for the upcoming year [employee_handbook-3.pdf].", - }, - "count": 106, -} - -assistant_message_perf_short = { - "message": { - "role": "assistant", - "content": "The supervisor will discuss the employee's performance and provide feedback on areas for improvement. They will also provide an opportunity for the employee to discuss their goals and objectives for the upcoming year. The review is a two-way dialogue between managers and employees, and employees will receive a written summary of their performance review which will include a rating of their performance, feedback, and goals for the upcoming year [employee_handbook-3.pdf].", - }, - "count": 91, -} - -user_message_dresscode = { - "message": { - "role": "user", - "content": "Is there a dress code?", - }, - "count": 13, -} - -assistant_message_dresscode = { - "message": { - "role": "assistant", - "content": "Yes, there is a dress code at Contoso Electronics. Look sharp! [employee_handbook-1.pdf]", - }, - "count": 30, -} - -user_message_pm = { - "message": { - "role": "user", - "content": "What does a Product Manager do?", - }, - "count": 14, -} - -text_and_image_message = { - "message": { - "role": "user", - "content": [ - {"type": "text", "text": "Describe this picture:"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg==", - "detail": "high", - }, - }, - ], - }, - "count": 266, -} - - -search_sources_toolchoice_auto = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "search_sources", - "description": "Retrieve sources from the Azure AI Search index", - "parameters": { - "type": "object", - "properties": { - "search_query": { - "type": "string", - "description": "Query string to retrieve documents from azure search eg: 'Health care plan'", - } - }, - "required": ["search_query"], - }, - }, - } - ], - "tool_choice": "auto", - "count": 66, -} - -search_sources_toolchoice_none = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "search_sources", - "description": "Retrieve sources from the Azure AI Search index", - "parameters": { - "type": "object", - "properties": { - "search_query": { - "type": "string", - "description": "Query string to retrieve documents from azure search eg: 'Health care plan'", - } - }, - "required": ["search_query"], - }, - }, - } - ], - "tool_choice": "none", - "count": 67, -} - -search_sources_toolchoice_name = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "search_sources", - "description": "Retrieve sources from the Azure AI Search index", - "parameters": { - "type": "object", - "properties": { - "search_query": { - "type": "string", - "description": "Query string to retrieve documents from azure search eg: 'Health care plan'", - } - }, - "required": ["search_query"], - }, - }, - } - ], - "tool_choice": {"type": "function", "function": {"name": "search_sources"}}, - "count": 75, -} - -integer_enum = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "data_demonstration", - "description": "This is the main function description", - "parameters": { - "type": "object", - "properties": { - "integer_enum": {"type": "integer", "enum": [-1, 1]} - }, - }, - }, - } - ], - "tool_choice": "none", - "count": 54, -} - - -integer_enum_tool_choice_name = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "data_demonstration", - "description": "This is the main function description", - "parameters": { - "type": "object", - "properties": { - "integer_enum": {"type": "integer", "enum": [-1, 1]} - }, - }, - }, - } - ], - "tool_choice": { - "type": "function", - "function": {"name": "data_demonstration"}, - }, # 4 tokens for "data_demonstration" - "count": 64, -} - -no_parameters = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "search_sources", - "description": "Retrieve sources from the Azure AI Search index", - }, - } - ], - "tool_choice": "auto", - "count": 42, -} - -no_parameters_tool_choice_name = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "search_sources", - "description": "Retrieve sources from the Azure AI Search index", - }, - } - ], - "tool_choice": { - "type": "function", - "function": {"name": "search_sources"}, - }, # 2 tokens for "search_sources" - "count": 51, -} - -no_parameter_description_or_required = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "search_sources", - "description": "Retrieve sources from the Azure AI Search index", - "parameters": { - "type": "object", - "properties": {"search_query": {"type": "string"}}, - }, - }, - } - ], - "tool_choice": "auto", - "count": 49, -} - -no_parameter_description = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "search_sources", - "description": "Retrieve sources from the Azure AI Search index", - "parameters": { - "type": "object", - "properties": {"search_query": {"type": "string"}}, - "required": ["search_query"], - }, - }, - } - ], - "tool_choice": "auto", - "count": 49, -} - -string_enum = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "summarize_order", - "description": "Summarize the customer order request", - "parameters": { - "type": "object", - "properties": { - "product_name": { - "type": "string", - "description": "Product name ordered by customer", - }, - "quantity": { - "type": "integer", - "description": "Quantity ordered by customer", - }, - "unit": { - "type": "string", - "enum": ["meals", "days"], - "description": "unit of measurement of the customer order", - }, - }, - "required": ["product_name", "quantity", "unit"], - }, - }, - } - ], - "tool_choice": "none", - "count": 86, -} - -inner_object = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "data_demonstration", - "description": "This is the main function description", - "parameters": { - "type": "object", - "properties": { - "object_1": { - "type": "object", - "description": "The object data type as a property", - "properties": { - "string1": {"type": "string"}, - }, - } - }, - "required": ["object_1"], - }, - }, - } - ], - "tool_choice": "none", - "count": 65, # counted 67, over by 2 -} -""" -namespace functions { - -// This is the main function description -type data_demonstration = (_: { -// The object data type as a property -object_1: { - string1?: string, -}, -}) => any; - -} // namespace functions -""" - -inner_object_with_enum_only = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "data_demonstration", - "description": "This is the main function description", - "parameters": { - "type": "object", - "properties": { - "object_1": { - "type": "object", - "description": "The object data type as a property", - "properties": { - "string_2a": { - "type": "string", - "enum": ["Happy", "Sad"], - } - }, - } - }, - "required": ["object_1"], - }, - }, - } - ], - "tool_choice": "none", - "count": 73, # counted 74, over by 1 -} -""" -namespace functions { - -// This is the main function description -type data_demonstration = (_: { -// The object data type as a property -object_1: { - string_2a?: "Happy" | "Sad", -}, -}) => any; - -} // namespace functions -""" - -inner_object_with_enum = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "data_demonstration", - "description": "This is the main function description", - "parameters": { - "type": "object", - "properties": { - "object_1": { - "type": "object", - "description": "The object data type as a property", - "properties": { - "string_2a": { - "type": "string", - "enum": ["Happy", "Sad"], - }, - "string_2b": { - "type": "string", - "description": "Description in a second object is lost", - }, - }, - } - }, - "required": ["object_1"], - }, - }, - } - ], - "tool_choice": "none", - "count": 89, # counted 92, over by 3 -} -""" -namespace functions { - -// This is the main function description -type data_demonstration = (_: { -// The object data type as a property -object_1: { - string_2a?: "Happy" | "Sad", - // Description in a second object is lost - string_2b?: string, -}, -}) => any; - -} // namespace functions -""" - -inner_object_and_string = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "data_demonstration", - "description": "This is the main function description", - "parameters": { - "type": "object", - "properties": { - "object_1": { - "type": "object", - "description": "The object data type as a property", - "properties": { - "string_2a": { - "type": "string", - "enum": ["Happy", "Sad"], - }, - "string_2b": { - "type": "string", - "description": "Description in a second object is lost", - }, - }, - }, - "string_1": { - "type": "string", - "description": "Not required gets a question mark", - }, - }, - "required": ["object_1"], - }, - }, - } - ], - "tool_choice": "none", - "count": 103, # counted 106, over by 3 -} -""" -namespace functions { - -// This is the main function description -type data_demonstration = (_: { -// The object data type as a property -object_1: { - string_2a?: "Happy" | "Sad", - // Description in a second object is lost - string_2b?: string, -}, -// Not required gets a question mark -string_1?: string, -}) => any; - -} // namespace functions -""" - -boolean = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "human_escalation", - "description": "Check if user wants to escalate to a human", - "parameters": { - "type": "object", - "properties": { - "requires_escalation": { - "type": "boolean", - "description": "If user is showing signs of frustration or anger in the query. Also if the user says they want to talk to a real person and not a chat bot.", - } - }, - "required": ["requires_escalation"], - }, - }, - } - ], - "tool_choice": "none", - "count": 89, # over by 3 -} - -array = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "get_coordinates", - "description": "Get the latitude and longitude of multiple mailing addresses", - "parameters": { - "type": "object", - "properties": { - "addresses": { - "type": "array", - "description": "The mailing addresses to be located", - "items": {"type": "string"}, - } - }, - "required": ["addresses"], - }, - }, - } - ], - "tool_choice": "none", - "count": 59, -} - -null = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "get_null", - "description": "Get the null value", - "parameters": { - "type": "object", - "properties": { - "null_value": { - "type": "null", - "description": "The null value to be returned", - } - }, - "required": ["null_value"], - }, - }, - } - ], - "tool_choice": "none", - "count": 55, -} - -no_type = { - "system_message": { - "role": "system", - "content": "You are a bot.", - }, - "tools": [ - { - "type": "function", - "function": { - "name": "get_no_type", - "description": "Get the no type value", - "parameters": { - "type": "object", - "properties": { - "no_type_value": { - "description": "The no type value to be returned", - } - }, - "required": ["no_type_value"], - }, - }, - } - ], - "tool_choice": "none", - "count": 59, -} - -MESSAGES_TEXT = [ - system_message, - system_message_short, - system_message_long, - system_message_unicode, - system_message_with_name, - user_message, - user_message_unicode, - user_message_perf, - user_message_dresscode, - user_message_pm, - assistant_message_perf, - assistant_message_perf_short, - assistant_message_dresscode, -] - -MESSAGES_WITH_IMAGES = [text_and_image_message] - -MESSAGES_WITH_TOOLS = [ - inner_object, - inner_object_and_string, - inner_object_with_enum_only, - inner_object_with_enum, - search_sources_toolchoice_auto, - search_sources_toolchoice_none, - search_sources_toolchoice_name, - integer_enum, - integer_enum_tool_choice_name, - no_parameters, - no_parameters_tool_choice_name, - no_parameter_description_or_required, - no_parameter_description, - string_enum, - boolean, - array, - no_type, - null, -] diff --git a/tests/proxy_unit_tests/model_cost.json b/tests/proxy_unit_tests/model_cost.json deleted file mode 100644 index 8d6f6851e16..00000000000 --- a/tests/proxy_unit_tests/model_cost.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "gpt-3.5-turbo": 7.7e-05 -} \ No newline at end of file diff --git a/tests/proxy_unit_tests/openai_batch_completions.jsonl b/tests/proxy_unit_tests/openai_batch_completions.jsonl deleted file mode 100644 index 05448952a0f..00000000000 --- a/tests/proxy_unit_tests/openai_batch_completions.jsonl +++ /dev/null @@ -1,2 +0,0 @@ -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} \ No newline at end of file diff --git a/tests/proxy_unit_tests/openai_batch_completions_router.jsonl b/tests/proxy_unit_tests/openai_batch_completions_router.jsonl deleted file mode 100644 index 8a4c99ca85e..00000000000 --- a/tests/proxy_unit_tests/openai_batch_completions_router.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"custom_id": "task-0", "method": "POST", "url": "/chat/completions", "body": {"model": "my-custom-name", "messages": [{"role": "system", "content": "You are an AI assistant that helps people find information."}, {"role": "user", "content": "When was Microsoft founded?"}]}} -{"custom_id": "task-1", "method": "POST", "url": "/chat/completions", "body": {"model": "my-custom-name", "messages": [{"role": "system", "content": "You are an AI assistant that helps people find information."}, {"role": "user", "content": "When was the first XBOX released?"}]}} -{"custom_id": "task-2", "method": "POST", "url": "/chat/completions", "body": {"model": "my-custom-name", "messages": [{"role": "system", "content": "You are an AI assistant that helps people find information."}, {"role": "user", "content": "What is Altair Basic?"}]}} \ No newline at end of file diff --git a/tests/proxy_unit_tests/speech_vertex.mp3 b/tests/proxy_unit_tests/speech_vertex.mp3 deleted file mode 100644 index c676110335e..00000000000 Binary files a/tests/proxy_unit_tests/speech_vertex.mp3 and /dev/null differ diff --git a/tests/proxy_unit_tests/test_proxy_custom_logger.py b/tests/proxy_unit_tests/test_proxy_custom_logger.py deleted file mode 100644 index 2516df2d58d..00000000000 --- a/tests/proxy_unit_tests/test_proxy_custom_logger.py +++ /dev/null @@ -1,319 +0,0 @@ -import sys, os -import traceback -from dotenv import load_dotenv - -load_dotenv() -import io, asyncio - -# this file is to test litellm/proxy - -import pytest, time -import litellm -from litellm import embedding, completion, completion_cost, Timeout -from litellm import RateLimitError -import importlib, inspect - -# test /chat/completion request to the proxy -from fastapi.testclient import TestClient -from fastapi import FastAPI -from litellm.proxy.proxy_server import ( - router, - save_worker_config, - initialize, -) # Replace with the actual module where your FastAPI router is defined - -filepath = os.path.dirname(os.path.abspath(__file__)) -python_file_path = f"{filepath}/test_configs/custom_callbacks.py" - - -@pytest.fixture -def client(): - filepath = os.path.dirname(os.path.abspath(__file__)) - config_fp = f"{filepath}/test_configs/test_custom_logger.yaml" - app = FastAPI() - asyncio.run(initialize(config=config_fp)) - app.include_router(router) # Include your router in the test app - return TestClient(app) - - -# Your bearer token -token = os.getenv("PROXY_MASTER_KEY") - -headers = {"Authorization": f"Bearer {token}"} - - -print("Testing proxy custom logger") - - -@pytest.mark.skipif( - os.environ.get("OPENAI_API_KEY") is None, - reason="OPENAI_API_KEY not set - skipping integration test", -) -def test_embedding(client): - try: - litellm.set_verbose = False - from litellm.proxy.types_utils.utils import get_instance_fn - - my_custom_logger = get_instance_fn( - value="custom_callbacks.my_custom_logger", config_file_path=python_file_path - ) - print("id of initialized custom logger", id(my_custom_logger)) - litellm.callbacks = [my_custom_logger] - # Your test data - print("initialized proxy") - # import the initialized custom logger - print(litellm.callbacks) - - # assert len(litellm.callbacks) == 1 # assert litellm is initialized with 1 callback - print("my_custom_logger", my_custom_logger) - assert my_custom_logger.async_success_embedding is False - - test_data = {"model": "azure-embedding-model", "input": ["hello"]} - response = client.post("/embeddings", json=test_data, headers=headers) - print("made request", response.status_code, response.text) - print( - "vars my custom logger /embeddings", - vars(my_custom_logger), - "id", - id(my_custom_logger), - ) - assert ( - my_custom_logger.async_success_embedding is True - ) # checks if the status of async_success is True, only the async_log_success_event can set this to true - assert ( - my_custom_logger.async_embedding_kwargs["model"] == "text-embedding-ada-002" - ) # checks if kwargs passed to async_log_success_event are correct - kwargs = my_custom_logger.async_embedding_kwargs - litellm_params = kwargs.get("litellm_params") - - # Test 1: Verify metadata is populated correctly - metadata = litellm_params.get("metadata", None) - print("\n\n Metadata in custom logger kwargs", litellm_params.get("metadata")) - assert metadata is not None, "metadata should be present in litellm_params" - assert "user_api_key" in metadata, "user_api_key should be in metadata" - assert "headers" in metadata, "headers should be in metadata" - - # Test 2: Verify proxy_server_request contains the original request details - proxy_server_request = litellm_params.get("proxy_server_request") - assert proxy_server_request is not None, "proxy_server_request should exist" - assert ( - proxy_server_request.get("url") == "http://testserver/embeddings" - ), "url should match" - assert proxy_server_request.get("method") == "POST", "method should be POST" - assert "headers" in proxy_server_request, "headers should be present" - assert "body" in proxy_server_request, "body should be present" - - # Test 3: Verify request body contains the original input data - body = proxy_server_request["body"] - assert ( - body.get("model") == "azure-embedding-model" - ), "model should match original request" - assert body.get("input") == ["hello"], "input should match original request" - - # Test 4: Verify model_info is populated - model_info = litellm_params.get("model_info") - assert model_info is not None, "model_info should exist" - assert model_info.get("mode") == "embedding", "mode should be embedding" - assert model_info.get("id") == "hello", "id should match" - assert ( - model_info.get("input_cost_per_token") == 0.002 - ), "input cost should match" - result = response.json() - print(f"Received response: {result}") - print("Passed Embedding custom logger on proxy!") - except Exception as e: - pytest.fail(f"LiteLLM Proxy test failed. Exception {str(e)}") - - -@pytest.mark.skipif( - os.environ.get("OPENAI_API_KEY") is None, - reason="OPENAI_API_KEY not set - skipping integration test", -) -def test_chat_completion(client): - try: - # Your test data - litellm.set_verbose = False - from litellm.proxy.types_utils.utils import get_instance_fn - - my_custom_logger = get_instance_fn( - value="custom_callbacks.my_custom_logger", config_file_path=python_file_path - ) - - print("id of initialized custom logger", id(my_custom_logger)) - - litellm.callbacks = [my_custom_logger] - # import the initialized custom logger - print(litellm.callbacks) - - # assert len(litellm.callbacks) == 1 # assert litellm is initialized with 1 callback - - print("LiteLLM Callbacks", litellm.callbacks) - print("my_custom_logger", my_custom_logger) - assert my_custom_logger.async_success == False - - test_data = { - "model": "Azure OpenAI GPT-4 Canada", - "messages": [ - {"role": "user", "content": "write a litellm poem"}, - ], - "max_tokens": 10, - } - - response = client.post("/chat/completions", json=test_data, headers=headers) - print("made request", response.status_code, response.text) - print("LiteLLM Callbacks", litellm.callbacks) - time.sleep(1) # sleep while waiting for callback to run - - print( - "my_custom_logger in /chat/completions", - my_custom_logger, - "id", - id(my_custom_logger), - ) - print("vars my custom logger, ", vars(my_custom_logger)) - assert ( - my_custom_logger.async_success == True - ) # checks if the status of async_success is True, only the async_log_success_event can set this to true - assert ( - my_custom_logger.async_completion_kwargs["model"] == "gpt-4.1-nano" - ) # checks if kwargs passed to async_log_success_event are correct - print( - "\n\n Custom Logger Async Completion args", - my_custom_logger.async_completion_kwargs, - ) - litellm_params = my_custom_logger.async_completion_kwargs.get("litellm_params") - - # Test 1: Verify metadata is populated correctly - metadata = litellm_params.get("metadata", None) - print("\n\n Metadata in custom logger kwargs", litellm_params.get("metadata")) - assert metadata is not None, "metadata should be present" - assert "user_api_key" in metadata, "user_api_key should be in metadata" - assert ( - "user_api_key_metadata" in metadata - ), "user_api_key_metadata should be in metadata" - assert "headers" in metadata, "headers should be in metadata" - - # Test 2: Verify model_info is populated - config_model_info = litellm_params.get("model_info") - assert config_model_info is not None, "model_info should exist" - assert config_model_info.get("id") == "gm", "model id should match" - assert config_model_info.get("mode") == "chat", "mode should be chat" - assert ( - config_model_info.get("input_cost_per_token") == 0.0002 - ), "input cost should match" - - # Test 3: Verify proxy_server_request contains request details - proxy_server_request_object = litellm_params.get("proxy_server_request") - assert ( - proxy_server_request_object is not None - ), "proxy_server_request should exist" - assert ( - proxy_server_request_object.get("url") - == "http://testserver/chat/completions" - ), "url should match" - assert ( - proxy_server_request_object.get("method") == "POST" - ), "method should be POST" - - # Test 4: Verify authorization is not leaked in logged headers - assert ( - "authorization" not in proxy_server_request_object["headers"] - ), "authorization should not be in headers" - - # Test 5: Verify request body contains original input data - body = proxy_server_request_object.get("body", {}) - assert ( - body.get("model") == "Azure OpenAI GPT-4 Canada" - ), "model should match original request" - assert body.get("messages") == [ - {"role": "user", "content": "write a litellm poem"} - ], "messages should match" - assert body.get("max_tokens") == 10, "max_tokens should match" - result = response.json() - print(f"Received response: {result}") - print("\nPassed /chat/completions with Custom Logger!") - except Exception as e: - pytest.fail(f"LiteLLM Proxy test failed. Exception {str(e)}") - - -@pytest.mark.skipif( - os.environ.get("OPENAI_API_KEY") is None, - reason="OPENAI_API_KEY not set - skipping integration test", -) -def test_chat_completion_stream(client): - try: - # Your test data - litellm.set_verbose = False - from litellm.proxy.types_utils.utils import get_instance_fn - - my_custom_logger = get_instance_fn( - value="custom_callbacks.my_custom_logger", config_file_path=python_file_path - ) - - print("id of initialized custom logger", id(my_custom_logger)) - - litellm.callbacks = [my_custom_logger] - import json - - print("initialized proxy") - # import the initialized custom logger - print(litellm.callbacks) - - print("LiteLLM Callbacks", litellm.callbacks) - print("my_custom_logger", my_custom_logger) - - assert ( - my_custom_logger.streaming_response_obj == None - ) # no streaming response obj is set pre call - - test_data = { - "model": "Azure OpenAI GPT-4 Canada", - "messages": [ - {"role": "user", "content": "write 1 line poem about LiteLLM"}, - ], - "max_tokens": 40, - "stream": True, # streaming call - } - - response = client.post("/chat/completions", json=test_data, headers=headers) - print("made request", response.status_code, response.text) - complete_response = "" - for line in response.iter_lines(): - if line: - # Process the streaming data line here - print("\n\n Line", line) - print(line) - line = str(line) - - json_data = line.replace("data: ", "") - - if "[DONE]" in json_data: - break - - # Parse the JSON string - data = json.loads(json_data) - - print("\n\n decode_data", data) - - # Access the content of choices[0]['message']['content'] - content = data["choices"][0]["delta"].get("content", None) or "" - - # Process the content as needed - print("Content:", content) - - complete_response += content - - print("\n\nHERE is the complete streaming response string", complete_response) - print("\n\nHERE IS the streaming Response from callback\n\n") - print(my_custom_logger.streaming_response_obj) - import time - - time.sleep(0.5) - - streamed_response = my_custom_logger.streaming_response_obj - assert ( - complete_response == streamed_response["choices"][0]["message"]["content"] - ) - - except Exception as e: - pytest.fail(f"LiteLLM Proxy test failed. Exception {str(e)}") diff --git a/tests/proxy_unit_tests/test_proxy_exception_mapping.py b/tests/proxy_unit_tests/test_proxy_exception_mapping.py deleted file mode 100644 index efaaa181600..00000000000 --- a/tests/proxy_unit_tests/test_proxy_exception_mapping.py +++ /dev/null @@ -1,319 +0,0 @@ -# test that the proxy actually does exception mapping to the OpenAI format - -import json -import os -from unittest import mock - -from dotenv import load_dotenv - -load_dotenv() -import asyncio -import io - -import openai -import pytest -from fastapi import Response -from fastapi.testclient import TestClient - -import litellm -from litellm.proxy.proxy_server import ( # Replace with the actual module where your FastAPI router is defined - initialize, - router, - save_worker_config, -) - -invalid_authentication_error_response = Response( - status_code=401, - content=json.dumps({"error": "Invalid Authentication"}), -) -context_length_exceeded_error_response_dict = { - "error": { - "message": "AzureException - Error code: 400 - {'error': {'message': \"This model's maximum context length is 4096 tokens. However, your messages resulted in 10007 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}", - "type": None, - "param": None, - "code": 400, - }, -} -context_length_exceeded_error_response = Response( - status_code=400, - content=json.dumps(context_length_exceeded_error_response_dict), -) - - -@pytest.fixture -def client(): - filepath = os.path.dirname(os.path.abspath(__file__)) - config_fp = f"{filepath}/test_configs/test_bad_config.yaml" - asyncio.run(initialize(config=config_fp)) - from litellm.proxy.proxy_server import app - - return TestClient(app) - - -# raise openai.AuthenticationError -def test_chat_completion_exception(client): - try: - # Your test data - test_data = { - "model": "gpt-3.5-turbo", - "messages": [ - {"role": "user", "content": "hi"}, - ], - "max_tokens": 10, - } - - response = client.post("/chat/completions", json=test_data) - - json_response = response.json() - print("keys in json response", json_response.keys()) - assert json_response.keys() == {"error"} - print("ERROR=", json_response["error"]) - assert isinstance(json_response["error"]["message"], str) - assert ( - "litellm.AuthenticationError: AuthenticationError" - in json_response["error"]["message"] - ) - - code_in_error = json_response["error"]["code"] - # OpenAI SDK required code to be STR, https://github.com/BerriAI/litellm/issues/4970 - # If we look on official python OpenAI lib, the code should be a string: - # https://github.com/openai/openai-python/blob/195c05a64d39c87b2dfdf1eca2d339597f1fce03/src/openai/types/shared/error_object.py#L11 - # Related LiteLLM issue: https://github.com/BerriAI/litellm/discussions/4834 - assert type(code_in_error) == str - - # make an openai client to call _make_status_error_from_response - openai_client = openai.OpenAI(api_key="anything") - openai_exception = openai_client._make_status_error_from_response( - response=response - ) - assert isinstance(openai_exception, openai.AuthenticationError) - - except Exception as e: - pytest.fail(f"LiteLLM Proxy test failed. Exception {str(e)}") - - -# raise openai.AuthenticationError -@mock.patch( - "litellm.proxy.proxy_server.llm_router.acompletion", - return_value=invalid_authentication_error_response, -) -def test_chat_completion_exception_azure(mock_acompletion, client): - try: - # Your test data - test_data = { - "model": "azure-gpt-3.5-turbo", - "messages": [ - {"role": "user", "content": "hi"}, - ], - "max_tokens": 10, - } - - response = client.post("/chat/completions", json=test_data) - - mock_acompletion.assert_called_once_with( - **test_data, - litellm_call_id=mock.ANY, - litellm_logging_obj=mock.ANY, - request_timeout=mock.ANY, - metadata=mock.ANY, - proxy_server_request=mock.ANY, - secret_fields=mock.ANY, - ) - - json_response = response.json() - print("keys in json response", json_response.keys()) - assert json_response.keys() == {"error"} - - # make an openai client to call _make_status_error_from_response - openai_client = openai.OpenAI(api_key="anything") - openai_exception = openai_client._make_status_error_from_response( - response=response - ) - print(openai_exception) - assert isinstance(openai_exception, openai.AuthenticationError) - - except Exception as e: - pytest.fail(f"LiteLLM Proxy test failed. Exception {str(e)}") - - -# raise openai.AuthenticationError -@mock.patch( - "litellm.proxy.proxy_server.llm_router.aembedding", - return_value=invalid_authentication_error_response, -) -def test_embedding_auth_exception_azure(mock_aembedding, client): - try: - # Your test data - test_data = {"model": "azure-embedding", "input": ["hi"]} - - response = client.post("/embeddings", json=test_data) - mock_aembedding.assert_called_once_with( - **test_data, - metadata=mock.ANY, - proxy_server_request=mock.ANY, - secret_fields=mock.ANY, - request_timeout=mock.ANY, - litellm_call_id=mock.ANY, - litellm_logging_obj=mock.ANY, - ) - print("Response from proxy=", response) - - json_response = response.json() - print("keys in json response", json_response.keys()) - assert json_response.keys() == {"error"} - - # make an openai client to call _make_status_error_from_response - openai_client = openai.OpenAI(api_key="anything") - openai_exception = openai_client._make_status_error_from_response( - response=response - ) - print("Exception raised=", openai_exception) - assert isinstance(openai_exception, openai.AuthenticationError) - - except Exception as e: - pytest.fail(f"LiteLLM Proxy test failed. Exception {str(e)}") - - -# raise openai.BadRequestError -# chat/completions openai -def test_exception_openai_bad_model(client): - try: - # Your test data - test_data = { - "model": "azure/GPT-12", - "messages": [ - {"role": "user", "content": "hi"}, - ], - "max_tokens": 10, - } - - response = client.post("/chat/completions", json=test_data) - - json_response = response.json() - print("keys in json response", json_response.keys()) - assert json_response.keys() == {"error"} - - # make an openai client to call _make_status_error_from_response - openai_client = openai.OpenAI(api_key="anything") - openai_exception = openai_client._make_status_error_from_response( - response=response - ) - print("Type of exception=", type(openai_exception)) - assert isinstance(openai_exception, openai.BadRequestError) - - except Exception as e: - pytest.fail(f"LiteLLM Proxy test failed. Exception {str(e)}") - - -# chat/completions any model -def test_chat_completion_exception_any_model(client): - try: - # Your test data - test_data = { - "model": "Lite-GPT-12", - "messages": [ - {"role": "user", "content": "hi"}, - ], - "max_tokens": 10, - } - - response = client.post("/chat/completions", json=test_data) - - json_response = response.json() - assert json_response.keys() == {"error"} - - # make an openai client to call _make_status_error_from_response - openai_client = openai.OpenAI(api_key="anything") - openai_exception = openai_client._make_status_error_from_response( - response=response - ) - assert isinstance(openai_exception, openai.BadRequestError) - _error_message = openai_exception.message - assert ( - "/chat/completions: Invalid model name passed in model=Lite-GPT-12" - in str(_error_message) - ) - - except Exception as e: - pytest.fail(f"LiteLLM Proxy test failed. Exception {str(e)}") - - -# embeddings any model -def test_embedding_exception_any_model(client): - try: - # Your test data - test_data = {"model": "Lite-GPT-12", "input": ["hi"]} - - response = client.post("/embeddings", json=test_data) - print("Response from proxy=", response) - print(response.json()) - - json_response = response.json() - print("keys in json response", json_response.keys()) - assert json_response.keys() == {"error"} - - # make an openai client to call _make_status_error_from_response - openai_client = openai.OpenAI(api_key="anything") - openai_exception = openai_client._make_status_error_from_response( - response=response - ) - print("Exception raised=", openai_exception) - assert isinstance(openai_exception, openai.BadRequestError) - _error_message = openai_exception.message - assert "/embeddings: Invalid model name passed in model=Lite-GPT-12" in str( - _error_message - ) - - except Exception as e: - pytest.fail(f"LiteLLM Proxy test failed. Exception {str(e)}") - - -# raise openai.BadRequestError -@mock.patch( - "litellm.proxy.proxy_server.llm_router.acompletion", - return_value=context_length_exceeded_error_response, -) -def test_chat_completion_exception_azure_context_window(mock_acompletion, client): - try: - # Your test data - test_data = { - "model": "working-azure-gpt-3.5-turbo", - "messages": [ - {"role": "user", "content": "hi" * 10000}, - ], - "max_tokens": 10, - } - response = None - - response = client.post("/chat/completions", json=test_data) - print("got response from server", response) - - mock_acompletion.assert_called_once_with( - **test_data, - litellm_call_id=mock.ANY, - litellm_logging_obj=mock.ANY, - request_timeout=mock.ANY, - metadata=mock.ANY, - proxy_server_request=mock.ANY, - secret_fields=mock.ANY, - ) - - json_response = response.json() - - print("keys in json response", json_response.keys()) - - assert json_response.keys() == {"error"} - - assert json_response == context_length_exceeded_error_response_dict - - # make an openai client to call _make_status_error_from_response - openai_client = openai.OpenAI(api_key="anything") - openai_exception = openai_client._make_status_error_from_response( - response=response - ) - print("exception from proxy", openai_exception) - assert isinstance(openai_exception, openai.BadRequestError) - print("passed exception is of type BadRequestError") - - except Exception as e: - pytest.fail(f"LiteLLM Proxy test failed. Exception {str(e)}") diff --git a/tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py b/tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py deleted file mode 100644 index 97d6b5c7cf3..00000000000 --- a/tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py +++ /dev/null @@ -1,50 +0,0 @@ -import os - -import pytest - - -@pytest.mark.skipif( - not os.getenv("GEMINI_API_KEY") and not os.getenv("GOOGLE_API_KEY"), - reason="Requires GEMINI_API_KEY or GOOGLE_API_KEY.", -) -@pytest.mark.asyncio -async def test_gemini_pass_through_endpoint(): - from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( - Request, - Response, - gemini_proxy_route, - ) - - body = b""" - { - "contents": [{ - "parts":[{ - "text": "The quick brown fox jumps over the lazy dog." - }] - }] - } - """ - - scope = { - "type": "http", - "method": "POST", - "path": "/gemini/v1beta/models/gemini-2.5-flash:countTokens", - "query_string": b"key=sk-9876", - "headers": [ - (b"content-type", b"application/json"), - ], - } - - async def async_receive(): - return {"type": "http.request", "body": body, "more_body": False} - - request = Request( - scope=scope, - receive=async_receive, - ) - - await gemini_proxy_route( - endpoint="v1beta/models/gemini-2.5-flash:countTokens?key=sk-9876", - request=request, - fastapi_response=Response(), - ) diff --git a/tests/router_unit_tests/README.md b/tests/router_unit_tests/README.md deleted file mode 100644 index 0206bcbb8f3..00000000000 --- a/tests/router_unit_tests/README.md +++ /dev/null @@ -1,5 +0,0 @@ -## Router component unit tests. - -Please name all files with the word 'router' in them. - -This is used to ensure all functions in the router are tested. \ No newline at end of file diff --git a/tests/router_unit_tests/conftest.py b/tests/router_unit_tests/conftest.py deleted file mode 100644 index cca1028aec7..00000000000 --- a/tests/router_unit_tests/conftest.py +++ /dev/null @@ -1,119 +0,0 @@ -# conftest.py - -import asyncio -import importlib - -import pytest - -import litellm # noqa: E402,F401 - -from tests._vcr_conftest_common import ( # noqa: E402,F401 - VerboseReporterState, - _pin_multipart_boundary, - apply_vcr_auto_marker_to_items, - emit_cassette_cache_session_banner, - emit_vcr_classification_summary, - emit_vcr_diagnostic_log, - install_live_call_probe, - record_vcr_outcome, - register_persister_if_enabled, - reset_vcr_diag_dir, - vcr_config_dict, -) - -_verbose_state = VerboseReporterState() - - -@pytest.fixture(scope="session") -def event_loop(): - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - yield loop - loop.close() - - -@pytest.fixture(scope="function", autouse=True) -def setup_and_teardown(): - """ - This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. - """ - - - from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER - - # flush all logs - asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue()) - - importlib.reload(litellm) - - try: - if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - importlib.reload(litellm.proxy.proxy_server) - except Exception as e: - print(f"Error reloading litellm.proxy.proxy_server: {e}") - - loop = asyncio.get_event_loop_policy().new_event_loop() - asyncio.set_event_loop(loop) - print(litellm) - yield - - # Teardown code (executes after the yield point) - loop.close() # Close the loop created earlier - asyncio.set_event_loop(None) # Remove the reference to the loop - - -@pytest.fixture(scope="module") -def vcr_config(): - return vcr_config_dict() - - -def pytest_recording_configure(config, vcr): - register_persister_if_enabled(vcr) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_makereport(item, call): - outcome = yield - rep = outcome.get_result() - setattr(item, f"rep_{rep.when}", rep) - - -@pytest.fixture(autouse=True) -def _vcr_outcome_gate(request, vcr): - install_live_call_probe(request, vcr) - yield - record_vcr_outcome(request, vcr) - - -def pytest_configure(config): - _verbose_state.remember_pluginmanager(config) - reset_vcr_diag_dir() - - -def pytest_runtest_logreport(report): - _verbose_state.maybe_emit_verdict(report) - - -def pytest_collection_modifyitems(config, items): - apply_vcr_auto_marker_to_items(items) - - # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests - custom_logger_tests = [ - item for item in items if "custom_logger" in item.parent.name - ] - other_tests = [item for item in items if "custom_logger" not in item.parent.name] - - # Sort tests based on their names - custom_logger_tests.sort(key=lambda x: x.name) - other_tests.sort(key=lambda x: x.name) - - # Reorder the items list - items[:] = custom_logger_tests + other_tests - - -def pytest_terminal_summary(terminalreporter, exitstatus, config): - emit_cassette_cache_session_banner(terminalreporter) - emit_vcr_classification_summary(terminalreporter) - emit_vcr_diagnostic_log(terminalreporter) diff --git a/tests/router_unit_tests/create_mock_standard_logging_payload.py b/tests/router_unit_tests/create_mock_standard_logging_payload.py deleted file mode 100644 index 096c8ff8c60..00000000000 --- a/tests/router_unit_tests/create_mock_standard_logging_payload.py +++ /dev/null @@ -1,128 +0,0 @@ -import io - - - -import asyncio -import gzip -import json -import logging -import time -from unittest.mock import AsyncMock, patch - -import pytest - -import litellm -from litellm import completion -from litellm._logging import verbose_logger -from litellm.integrations.datadog.datadog import * -from datetime import datetime, timedelta -from litellm.types.utils import ( - StandardLoggingPayload, - StandardLoggingModelInformation, - StandardLoggingMetadata, - StandardLoggingHiddenParams, -) - -verbose_logger.setLevel(logging.DEBUG) - - -def create_standard_logging_payload() -> StandardLoggingPayload: - return StandardLoggingPayload( - id="test_id", - call_type="completion", - response_cost=0.1, - response_cost_failure_debug_info=None, - status="success", - total_tokens=30, - prompt_tokens=20, - completion_tokens=10, - startTime=1234567890.0, - endTime=1234567891.0, - completionStartTime=1234567890.5, - model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-5-mini", model_map_value=None - ), - model="gpt-5-mini", - model_id="model-123", - model_group="openai-gpt", - api_base="https://api.openai.com", - metadata=StandardLoggingMetadata( - user_api_key_hash="test_hash", - user_api_key_org_id=None, - user_api_key_alias="test_alias", - user_api_key_team_id="test_team", - user_api_key_user_id="test_user", - user_api_key_team_alias="test_team_alias", - spend_logs_metadata=None, - requester_ip_address="127.0.0.1", - requester_metadata=None, - ), - cache_hit=False, - cache_key=None, - saved_cache_cost=0.0, - request_tags=[], - end_user=None, - requester_ip_address="127.0.0.1", - messages=[{"role": "user", "content": "Hello, world!"}], - response={"choices": [{"message": {"content": "Hi there!"}}]}, - error_str=None, - model_parameters={"stream": True}, - hidden_params=StandardLoggingHiddenParams( - model_id="model-123", - cache_key=None, - api_base="https://api.openai.com", - response_cost="0.1", - additional_headers=None, - ), - ) - - -def create_standard_logging_payload_with_long_content() -> StandardLoggingPayload: - return StandardLoggingPayload( - id="test_id", - call_type="completion", - response_cost=0.1, - response_cost_failure_debug_info=None, - status="success", - total_tokens=30, - prompt_tokens=20, - completion_tokens=10, - startTime=1234567890.0, - endTime=1234567891.0, - completionStartTime=1234567890.5, - model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-5-mini", model_map_value=None - ), - model="gpt-5-mini", - model_id="model-123", - model_group="openai-gpt", - api_base="https://api.openai.com", - metadata=StandardLoggingMetadata( - user_api_key_hash="test_hash", - user_api_key_org_id=None, - user_api_key_alias="test_alias", - user_api_key_team_id="test_team", - user_api_key_user_id="test_user", - user_api_key_team_alias="test_team_alias", - spend_logs_metadata=None, - requester_ip_address="127.0.0.1", - requester_metadata=None, - ), - cache_hit=False, - cache_key=None, - saved_cache_cost=0.0, - request_tags=[], - end_user=None, - requester_ip_address="127.0.0.1", - messages=[{"role": "user", "content": "Hello, world!" * 80000}], - response={"choices": [{"message": {"content": "Hi there!" * 80000}}]}, - error_str="error_str" * 80000, - model_parameters={"stream": True}, - hidden_params=StandardLoggingHiddenParams( - model_id="model-123", - cache_key=None, - api_base="https://api.openai.com", - response_cost="0.1", - additional_headers=None, - ), - ) diff --git a/tests/router_unit_tests/test_router_prompt_caching.py b/tests/router_unit_tests/test_router_prompt_caching.py deleted file mode 100644 index 329ea9f3931..00000000000 --- a/tests/router_unit_tests/test_router_prompt_caching.py +++ /dev/null @@ -1,136 +0,0 @@ -import asyncio - -import pytest - -from litellm import Router -from litellm.router_utils.prompt_caching_cache import PromptCachingCache - - -@pytest.mark.asyncio -async def test_router_prompt_caching_same_cacheable_prefix_routes_to_same_deployment(): - """ - End-to-end test to validate prompt caching routing through LiteLLM Router. - - Tests that requests with same cacheable content but different user messages - route to the same deployment (for prompt caching). - - This reproduces the issue where requests with same cacheable prefix but different - user messages should route to the same deployment, but previously didn't because - the cache key included the entire messages array instead of just the cacheable prefix. - """ - from litellm.types.llms.openai import AllMessageValues - - def create_messages(user_content: str) -> list[AllMessageValues]: - """ - Create messages matching the user's exact scenario. - - Message structure: - - BLOCK 1: System message, first content block (no cache_control) - → INCLUDED (comes before the last cacheable block) - - BLOCK 2: System message, second content block (WITH cache_control) - → INCLUDED (this IS the last cacheable block) - - USER MESSAGE: User message (no cache_control) - → NOT included (comes after last cacheable block) - """ - return [ - { - "role": "system", - "content": [ - # BLOCK 1: No cache_control → INCLUDED (all blocks up to last cacheable are included) - { - "type": "text", - "text": "You are an AI assistant tasked with analyzing legal documents.", - }, - # BLOCK 2: Has cache_control → INCLUDED (this is the last cacheable block) - { - "type": "text", - "text": "Here 3 is the full text of a complex legal agreement" - * 400, - "cache_control": {"type": "ephemeral"}, - }, - ], - }, - { - "role": "user", - # USER MESSAGE: No cache_control → NOT included (comes after last cacheable block) - "content": user_content, - }, - ] - - # Create router with multiple deployments - router = Router( - model_list=[ - { - "model_name": "test-model", - "litellm_params": { - "model": "gpt-5-mini", - "api_base": "https://exampleopenaiendpoint-production-0ee2.up.railway.app/v1", - "api_key": f"test-key-{i}", - }, - "model_info": {"id": f"deployment-{i}"}, - } - for i in range(1, 4) - ], - routing_strategy="simple-shuffle", - optional_pre_call_checks=["prompt_caching"], - ) - - # Create test messages matching user's exact scenario - # Same cacheable prefix (system blocks 1+2) but different user messages - messages1 = create_messages( - "what are the key terms and conditions in this agreement?" - ) - messages2 = create_messages("how many words are there?") - messages3 = create_messages("how many sentences are there?") - - cache = PromptCachingCache(cache=router.cache) - - # Test 1: Cache keys should be same (same cacheable prefix, different user messages) - key1 = PromptCachingCache.get_prompt_caching_cache_key(messages1, None) - key2 = PromptCachingCache.get_prompt_caching_cache_key(messages2, None) - key3 = PromptCachingCache.get_prompt_caching_cache_key(messages3, None) - - assert key1 is not None, "Cache key should not be None" - assert ( - key1 == key2 == key3 - ), "Cache keys should be the same for same cacheable prefix" - - # Make first request - try: - response1 = await router.acompletion(model="test-model", messages=messages1) - model_id_1 = response1._hidden_params.get("model_id", "unknown") - except Exception: - # If API call fails, we can still test the cache key logic - model_id_1 = "unknown" - - await asyncio.sleep(1) # Wait for cache write - - # Test 2: Cache lookup should work for messages2 (same cacheable prefix) - cached_2 = await cache.async_get_model_id(messages2, None) - # Cache should be found if first request succeeded - if model_id_1 != "unknown": - assert ( - cached_2 is not None - ), "Cache lookup should work for same cacheable prefix" - - # Make second request - try: - response2 = await router.acompletion(model="test-model", messages=messages2) - model_id_2 = response2._hidden_params.get("model_id", "unknown") - except Exception: - model_id_2 = "unknown" - - await asyncio.sleep(1) # Wait for cache write - - # Make third request - try: - response3 = await router.acompletion(model="test-model", messages=messages3) - model_id_3 = response3._hidden_params.get("model_id", "unknown") - except Exception: - model_id_3 = "unknown" - - # Test 3: All requests should route to same deployment (if API calls succeeded) - if model_id_1 != "unknown" and model_id_2 != "unknown" and model_id_3 != "unknown": - assert ( - model_id_1 == model_id_2 == model_id_3 - ), f"All requests should route to same deployment, but got: {model_id_1}, {model_id_2}, {model_id_3}" diff --git a/tests/test_litellm/__init__.py b/tests/test_litellm/__init__.py deleted file mode 100644 index 7f5464af889..00000000000 --- a/tests/test_litellm/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# This file makes the tests/litellm directory a Python package diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py deleted file mode 100644 index bd38d3b02e8..00000000000 --- a/tests/test_litellm/conftest.py +++ /dev/null @@ -1,598 +0,0 @@ -# conftest.py - IMPROVED VERSION -# -# Key changes: -# 1. Changed module reload from 'module' scope to 'function' scope for better isolation -# 2. Made cache flushing happen per-function instead of per-module -# 3. Removed manual event loop creation (let pytest-asyncio handle it) -# 4. Added proper cleanup in fixtures -# 5. Added worker-specific isolation for parallel execution - -import base64 -import importlib -import os -from pathlib import Path -from types import SimpleNamespace -import httpx -import pytest -from pytest_socket import _remove_restrictions - -import asyncio - -import litellm -from litellm import router as litellm_router_module -from litellm import utils as litellm_utils_module -from litellm._logging import ALL_LOGGERS -from litellm.litellm_core_utils.prompt_templates import ( - image_handling as image_handling_module, -) -from litellm.llms.custom_httpx.async_client_cleanup import ( - close_litellm_async_clients, -) -from litellm.proxy.db import tool_registry_writer as tool_registry_writer_module -from tests.unit.litellm_core_utils.fake_secret_vault import FakeSecretVault - - -def _reset_module_level_aws_auth_caches(): - """ - Clear module-level AWS auth state that can survive between tests. - - Bedrock/SageMaker handlers are instantiated once at import time and cache - resolved credentials on the handler instance. If a previous test resolves an - invalid or different auth flow, later tests can reuse that cached state and - bypass their local monkeypatched env setup. - """ - for module_name in ( - "litellm.main", - "litellm.files.main", - "litellm.rerank_api.main", - "litellm.realtime_api.main", - ): - try: - module = importlib.import_module(module_name) - except Exception: - continue - for attr_name in dir(module): - obj = getattr(module, attr_name) - iam_cache = getattr(obj, "iam_cache", None) - if iam_cache is None: - continue - flush_cache = getattr(iam_cache, "flush_cache", None) - if callable(flush_cache): - flush_cache() - - try: - import boto3 - - boto3.DEFAULT_SESSION = None - except Exception: - pass - - -@pytest.fixture(scope="session") -def isolated_aws_credentials_dir(tmp_path_factory): - aws_dir = tmp_path_factory.mktemp("aws-config") - credentials_file = Path(aws_dir) / "credentials" - config_file = Path(aws_dir) / "config" - credentials_file.write_text("", encoding="utf-8") - config_file.write_text("", encoding="utf-8") - return { - "credentials": str(credentials_file), - "config": str(config_file), - } - - -@pytest.fixture(scope="function", autouse=True) -def isolate_host_aws_config(monkeypatch, isolated_aws_credentials_dir): - """Prevent botocore from reading host AWS profiles during unit tests.""" - monkeypatch.setenv( - "AWS_SHARED_CREDENTIALS_FILE", isolated_aws_credentials_dir["credentials"] - ) - monkeypatch.setenv("AWS_CONFIG_FILE", isolated_aws_credentials_dir["config"]) - monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true") - monkeypatch.delenv("AWS_PROFILE", raising=False) - monkeypatch.delenv("AWS_DEFAULT_PROFILE", raising=False) - monkeypatch.delenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", raising=False) - monkeypatch.delenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", raising=False) - monkeypatch.delenv("AWS_SESSION_TOKEN", raising=False) - monkeypatch.delenv("AWS_ROLE_ARN", raising=False) - monkeypatch.delenv("AWS_WEB_IDENTITY_TOKEN_FILE", raising=False) - monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) - monkeypatch.delenv("AWS_REGION_NAME", raising=False) - monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False) - - -@pytest.fixture(scope="function", autouse=True) -def isolate_host_proxy_base_url(monkeypatch): - """Prevent a host PROXY_BASE_URL from outranking request-derived URLs during unit tests.""" - monkeypatch.delenv("PROXY_BASE_URL", raising=False) - - -@pytest.fixture(scope="function", autouse=True) -def isolate_host_os_keychain(monkeypatch): - """Keep any code path that resolves a CLI credential out of the developer's real OS keychain. - - Tests that exercise keychain behaviour inject their own vault instead. - """ - monkeypatch.setenv("LITELLM_CLI_DISABLE_KEYRING", "1") - - -@pytest.fixture -def secret_vault_factory(): - """Build FakeSecretVault instances; see its docstring for the failure modes it can model.""" - return FakeSecretVault - - -@pytest.fixture -def local_model_cost_map(monkeypatch): - """Force the bundled in-repo cost map so capability and pricing assertions do not - depend on the network-fetched ``main`` copy, which lags this branch until merge. - - ``get_model_info`` is lru_cached, so swapping ``model_cost`` is not enough on its - own; clear on the way in and out so entries warmed against either map never leak - across tests.""" - original_model_cost = litellm.model_cost - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - litellm.get_model_info.cache_clear() - try: - yield - finally: - litellm.model_cost = original_model_cost - litellm.get_model_info.cache_clear() - - -@pytest.fixture -def local_beta_headers_config(monkeypatch): - """Pin the bundled ``anthropic_beta_headers_config.json`` so beta header assertions - do not depend on the network-fetched copy or on what earlier tests left cached.""" - from litellm.anthropic_beta_headers_manager import reload_beta_headers_config - - monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True") - reload_beta_headers_config() - try: - yield - finally: - monkeypatch.delenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", raising=False) - reload_beta_headers_config() - - -def _run_coroutine_if_needed(result): - if not asyncio.iscoroutine(result): - return - - try: - asyncio.run(result) - except RuntimeError: - # If pytest-asyncio already has a running loop, best-effort scheduling is - # still better than leaking the client entirely. - try: - loop = asyncio.get_running_loop() - except RuntimeError: - return - if loop.is_running(): - loop.create_task(result) - except Exception: - pass - - -def _close_handler_if_needed(handler): - if handler is None: - return - - close_fn = getattr(handler, "close", None) - if not callable(close_fn): - return - - try: - result = close_fn() - _run_coroutine_if_needed(result) - except Exception: - pass - - -@pytest.fixture(scope="function", autouse=True) -def isolate_litellm_state(): - """ - Per-function isolation fixture (changed from module scope). - - This ensures better isolation when running tests in parallel: - - Each test function gets a clean litellm state - - Cache is flushed before each test - - No module reloading during parallel execution - - Note: Module reloading at function scope is safer for parallel execution - but adds overhead. Consider removing reload entirely if tests can work without it. - """ - # Get worker ID if running with pytest-xdist - worker_id = os.environ.get("PYTEST_XDIST_WORKER", "master") - - # Store original callback state (all callback lists) - original_state = {} - if hasattr(litellm, "callbacks"): - original_state["callbacks"] = ( - litellm.callbacks.copy() if litellm.callbacks else [] - ) - if hasattr(litellm, "success_callback"): - original_state["success_callback"] = ( - litellm.success_callback.copy() if litellm.success_callback else [] - ) - if hasattr(litellm, "failure_callback"): - original_state["failure_callback"] = ( - litellm.failure_callback.copy() if litellm.failure_callback else [] - ) - if hasattr(litellm, "input_callback"): - original_state["input_callback"] = ( - litellm.input_callback.copy() if litellm.input_callback else [] - ) - if hasattr(litellm, "_async_success_callback"): - original_state["_async_success_callback"] = ( - litellm._async_success_callback.copy() - if litellm._async_success_callback - else [] - ) - if hasattr(litellm, "_async_failure_callback"): - original_state["_async_failure_callback"] = ( - litellm._async_failure_callback.copy() - if litellm._async_failure_callback - else [] - ) - if hasattr(litellm, "_async_input_callback"): - original_state["_async_input_callback"] = ( - litellm._async_input_callback.copy() - if litellm._async_input_callback - else [] - ) - - # Store routing globals — leaked model_fallbacks causes tests to route - # through async_completion_with_fallbacks / Router, bypassing HTTP mocks - if hasattr(litellm, "model_fallbacks"): - original_state["model_fallbacks"] = litellm.model_fallbacks - - # Store transport/network globals — many tests set these without restoring, - # causing subsequent tests to get None from create_async_transport() - for _attr in ("disable_aiohttp_transport", "force_ipv4"): - if hasattr(litellm, _attr): - original_state[_attr] = getattr(litellm, _attr) - - # Store request-mapping globals that are frequently mutated in tests. - if hasattr(litellm, "drop_params"): - original_state["drop_params"] = litellm.drop_params - if hasattr(litellm, "cache"): - original_state["cache"] = litellm.cache - - # Store secret-manager globals. Several tests swap these out, which changes - # get_secret() behavior for later env-driven tests (for example Redis config). - for _attr in ( - "secret_manager_client", - "_key_management_system", - "_key_management_settings", - ): - if hasattr(litellm, _attr): - original_state[_attr] = getattr(litellm, _attr) - - # Store other commonly-mutated LiteLLM globals that affect provider routing, - # auth, and request shaping during larger suite runs. - for _attr in ( - "api_base", - "num_retries", - "modify_params", - "ssl_verify", - "credential_list", - "model_group_settings", - "default_internal_user_params", - "default_team_params", - "prometheus_emit_stream_label", - "vector_store_registry", - "model_cost", - "cost_margin_config", - "cost_discount_config", - "disable_hf_tokenizer_download", - "disable_copilot_system_to_assistant", - "cohere_models", - "anthropic_models", - "token_counter", - "initialized_langfuse_clients", - ): - if hasattr(litellm, _attr): - original_state[_attr] = getattr(litellm, _attr) - - original_runtime_registered_model_cost = { - model_key: dict(model_value) - for model_key, model_value in litellm_utils_module._runtime_registered_model_cost.items() - } - - original_live_routers = set(litellm_router_module._live_routers) - - # Store LiteLLM logger state. Some tests reconfigure handlers/propagation for - # JSON logging and do not restore them, which breaks later caplog-based tests. - logger_state = {} - for logger in ALL_LOGGERS: - logger_state[logger.name] = { - "level": logger.level, - "disabled": logger.disabled, - "propagate": logger.propagate, - "handlers": list(logger.handlers), - "filters": list(logger.filters), - } - - # Store singleton registries that are lazily initialized during tests and - # can change endpoint behavior later in the suite. - original_tool_policy_registry = tool_registry_writer_module._tool_policy_registry - had_module_level_client = "module_level_client" in litellm.__dict__ - had_module_level_aclient = "module_level_aclient" in litellm.__dict__ - original_module_level_client = litellm.__dict__.get("module_level_client") - original_module_level_aclient = litellm.__dict__.get("module_level_aclient") - - # Flush cache before test (critical for respx mocks) - if hasattr(litellm, "in_memory_llm_clients_cache"): - litellm.in_memory_llm_clients_cache.flush_cache() - image_handling_module.in_memory_cache.flush_cache() - _reset_module_level_aws_auth_caches() - # litellm.get_model_info() memoizes ModelInfo built from litellm.model_cost, so a - # test that rebinds the cost map leaves later tests pricing against the old map. - litellm_utils_module._invalidate_model_cost_lowercase_map() - - # Clear all callback lists to prevent cross-test contamination - if hasattr(litellm, "callbacks"): - litellm.callbacks = [] - if hasattr(litellm, "success_callback"): - litellm.success_callback = [] - if hasattr(litellm, "failure_callback"): - litellm.failure_callback = [] - if hasattr(litellm, "input_callback"): - litellm.input_callback = [] - if hasattr(litellm, "_async_success_callback"): - litellm._async_success_callback = [] - if hasattr(litellm, "_async_failure_callback"): - litellm._async_failure_callback = [] - if hasattr(litellm, "_async_input_callback"): - litellm._async_input_callback = [] - - # Clear routing globals - if hasattr(litellm, "model_fallbacks"): - litellm.model_fallbacks = None - if hasattr(litellm, "cache"): - litellm.cache = None - litellm.__dict__.pop("module_level_client", None) - litellm.__dict__.pop("module_level_aclient", None) - tool_registry_writer_module._tool_policy_registry = None - - yield - - # Cleanup after test - if hasattr(litellm, "in_memory_llm_clients_cache"): - litellm.in_memory_llm_clients_cache.flush_cache() - image_handling_module.in_memory_cache.flush_cache() - _reset_module_level_aws_auth_caches() - current_module_level_client = litellm.__dict__.get("module_level_client") - current_module_level_aclient = litellm.__dict__.get("module_level_aclient") - - # Restore all callback lists to original state - for attr_name, original_value in original_state.items(): - if hasattr(litellm, attr_name): - setattr(litellm, attr_name, original_value) - - litellm_utils_module._runtime_registered_model_cost.clear() - litellm_utils_module._runtime_registered_model_cost.update(original_runtime_registered_model_cost) - litellm_utils_module._invalidate_model_cost_lowercase_map() - - for _router in tuple(litellm_router_module._live_routers): - litellm_router_module._live_routers.discard(_router) - for _router in original_live_routers: - litellm_router_module._live_routers.add(_router) - - # Restore logger configuration mutated by logging-focused tests. - for logger in ALL_LOGGERS: - original_logger_state = logger_state.get(logger.name) - if original_logger_state is None: - continue - logger.setLevel(original_logger_state["level"]) - logger.disabled = original_logger_state["disabled"] - logger.propagate = original_logger_state["propagate"] - logger.handlers = list(original_logger_state["handlers"]) - logger.filters = list(original_logger_state["filters"]) - - tool_registry_writer_module._tool_policy_registry = original_tool_policy_registry - if current_module_level_client is not original_module_level_client: - _close_handler_if_needed(current_module_level_client) - if current_module_level_aclient is not original_module_level_aclient: - _close_handler_if_needed(current_module_level_aclient) - if had_module_level_client: - litellm.__dict__["module_level_client"] = original_module_level_client - else: - litellm.__dict__.pop("module_level_client", None) - if had_module_level_aclient: - litellm.__dict__["module_level_aclient"] = original_module_level_aclient - else: - litellm.__dict__.pop("module_level_aclient", None) - - -@pytest.fixture(scope="module", autouse=True) -def setup_and_teardown(): - """ - Module-scoped setup/teardown for heavy initialization. - - Use this sparingly - most state should be handled by isolate_litellm_state. - Only reload modules here if absolutely necessary. - """ - - import litellm - - # Only reload if NOT running in parallel (module reload + parallel = bad) - worker_id = os.environ.get("PYTEST_XDIST_WORKER", None) - if worker_id is None: - # Single process mode - safe to reload - importlib.reload(litellm) - - try: - if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - import litellm.proxy.proxy_server - - importlib.reload(litellm.proxy.proxy_server) - except Exception as e: - print(f"Error reloading litellm.proxy.proxy_server: {e}") - - # Flush cache after reload (prevents stale client instances) - if hasattr(litellm, "in_memory_llm_clients_cache"): - litellm.in_memory_llm_clients_cache.flush_cache() - - print(f"[conftest] Module setup complete (worker: {worker_id or 'master'})") - - yield - - # Teardown - no need to manually manage event loops with pytest-asyncio auto mode - print(f"[conftest] Module teardown complete (worker: {worker_id or 'master'})") - - -def pytest_collectstart(): - _remove_restrictions() - - -def pytest_runtest_setup(): - _remove_restrictions() - - -def pytest_collection_modifyitems(config, items): - """ - Customize test collection order. - - - Separate tests marked with 'no_parallel' from parallelizable tests - - Sort custom_logger tests first (they tend to interfere with other tests) - """ - # Separate no_parallel tests - no_parallel_tests = [ - item - for item in items - if any(mark.name == "no_parallel" for mark in item.iter_markers()) - ] - - # Separate custom_logger tests - custom_logger_tests = [ - item - for item in items - if "custom_logger" in item.parent.name and item not in no_parallel_tests - ] - - # Everything else - other_tests = [ - item - for item in items - if item not in no_parallel_tests and item not in custom_logger_tests - ] - - # Sort each group - custom_logger_tests.sort(key=lambda x: x.name) - other_tests.sort(key=lambda x: x.name) - no_parallel_tests.sort(key=lambda x: x.name) - - # Reorder: custom_logger first (isolated), then other tests, then no_parallel tests last - items[:] = custom_logger_tests + other_tests + no_parallel_tests - - -def pytest_configure(config): - """ - Configure pytest with custom settings. - """ - # Add marker for flaky tests (for documentation purposes) - config.addinivalue_line( - "markers", "flaky: mark test as potentially flaky (should use --reruns)" - ) - - # Detect if running in CI - is_ci = os.environ.get("CI") == "true" or os.environ.get("LITELLM_CI") == "true" - if is_ci: - print("[conftest] Running in CI mode - enabling stricter test isolation") - - -# Optional: Add a fixture for tests that need even stricter isolation -@pytest.fixture -def strict_isolation(): - """ - Use this fixture for tests that need extra strict isolation. - - Example: - def test_something(strict_isolation): - # Test code with guaranteed clean state - pass - """ - # Force flush all caches - if hasattr(litellm, "in_memory_llm_clients_cache"): - litellm.in_memory_llm_clients_cache.flush_cache() - - # Reset all global state - if hasattr(litellm, "disable_aiohttp_transport"): - original_aiohttp = litellm.disable_aiohttp_transport - litellm.disable_aiohttp_transport = False - else: - original_aiohttp = None - - if hasattr(litellm, "set_verbose"): - original_verbose = litellm.set_verbose - litellm.set_verbose = False - else: - original_verbose = None - - yield - - # Restore original state - if original_aiohttp is not None: - litellm.disable_aiohttp_transport = original_aiohttp - if original_verbose is not None: - litellm.set_verbose = original_verbose - - # Final cache flush - if hasattr(litellm, "in_memory_llm_clients_cache"): - litellm.in_memory_llm_clients_cache.flush_cache() - - -def pytest_sessionfinish(session, exitstatus): - """Close any globally cached HTTP clients so xdist workers exit cleanly.""" - _close_handler_if_needed(litellm.__dict__.get("module_level_client")) - _close_handler_if_needed(litellm.__dict__.get("module_level_aclient")) - litellm.__dict__.pop("module_level_client", None) - litellm.__dict__.pop("module_level_aclient", None) - _close_handler_if_needed(getattr(litellm, "base_llm_aiohttp_handler", None)) - _close_handler_if_needed(getattr(litellm, "httpx_client", None)) - _close_handler_if_needed(getattr(litellm, "aclient", None)) - _close_handler_if_needed(getattr(litellm, "client", None)) - _run_coroutine_if_needed(close_litellm_async_clients()) - - -ONE_PIXEL_PNG = base64.b64decode( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" -) - - -@pytest.fixture -def async_only_image_fetch(monkeypatch): - from litellm.litellm_core_utils.prompt_templates import factory, image_handling - from litellm.llms.gemini.chat import transformation as gemini_chat_transformation - - fetch = SimpleNamespace( - fetched=[], - base64_png=base64.b64encode(ONE_PIXEL_PNG).decode(), - data_url="data:image/png;base64," + base64.b64encode(ONE_PIXEL_PNG).decode(), - ) - - def forbid_sync_fetch(client, url, **kwargs): - raise litellm.ImageFetchError(f"sync image fetch ran on the event loop: {url}") - - async def serve_png(client, url, **kwargs): - fetch.fetched.append(url) - return httpx.Response( - 200, - content=ONE_PIXEL_PNG, - headers={"content-type": "image/png"}, - request=httpx.Request("GET", url), - ) - - def forbid_sync_convert(url, *args, **kwargs): - if url.startswith(("http://", "https://")): - raise litellm.ImageFetchError(f"sync convert_url_to_base64 ran on the request path: {url}") - return url - - monkeypatch.setattr(image_handling, "safe_get", forbid_sync_fetch) - monkeypatch.setattr(image_handling, "async_safe_get", serve_png) - for module in (image_handling, factory, gemini_chat_transformation): - monkeypatch.setattr(module, "convert_url_to_base64", forbid_sync_convert) - return fetch diff --git a/tests/test_litellm/readme.md b/tests/test_litellm/readme.md deleted file mode 100644 index d126ea8e75b..00000000000 --- a/tests/test_litellm/readme.md +++ /dev/null @@ -1,24 +0,0 @@ -# Testing for `litellm/` - -This directory 1:1 maps the the `litellm/` directory, and can only contain mocked tests. - -The point of this is to: -1. Increase test coverage of `litellm/` -2. Make it easy for contributors to add tests for the `litellm/` package and easily run tests without needing LLM API keys. - - -## File name conventions - -- `litellm/proxy/test_caching_routes.py` maps to `litellm/proxy/caching_routes.py` -- `test_.py` maps to `litellm/.py` - - - - - - - - - - - diff --git a/tests/unit/a2a_protocol/test_completion_bridge_streaming.py b/tests/unit/a2a_protocol/test_completion_bridge_streaming.py index 913c917bd2d..5ed2cf0aeb6 100644 --- a/tests/unit/a2a_protocol/test_completion_bridge_streaming.py +++ b/tests/unit/a2a_protocol/test_completion_bridge_streaming.py @@ -363,3 +363,79 @@ async def test_handle_streaming_keeps_agent_card_path_out_of_the_completion_call assert len(events) == 4 assert "agent_card_path" not in mock_acompletion.call_args.kwargs + + +AGENTCORE_ARN = "arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/unit-agent-abc123" +AGENTCORE_INVOCATIONS_URL = ( + "https://bedrock-agentcore.us-west-2.amazonaws.com/runtimes/" + + AGENTCORE_ARN.replace(":", "%3A").replace("/", "%2F") + + "/invocations" +) + + +@pytest.mark.asyncio +async def test_a2a_completion_bridge_bedrock_agentcore(respx_mock, monkeypatch): + """ + A2A streaming via the completion bridge against a scripted Bedrock AgentCore + upstream: asserts the outbound invocation request and the emitted A2A event + sequence. + """ + import litellm + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + import httpx + from a2a.compat.v0_3.types import MessageSendParams, SendStreamingMessageRequest + from uuid import uuid4 + + from litellm.a2a_protocol import asend_message_streaming + + sse_body = ( + 'data: {"id":"chatcmpl-1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"Machine "}}]}\n\n' + 'data: {"id":"chatcmpl-1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"learning."}}]}\n\n' + 'data: {"id":"chatcmpl-1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n' + "data: [DONE]\n\n" + ) + invoke_route = respx_mock.post( + url__regex=r"https://bedrock-agentcore\.us-west-2\.amazonaws\.com/runtimes/.*/invocations.*" + ).mock( + return_value=httpx.Response(200, content=sse_body.encode(), headers={"content-type": "text/event-stream"}) + ) + + send_message_payload = { + "message": { + "role": "user", + "parts": [{"kind": "text", "text": "Explain machine learning in simple terms"}], + "messageId": uuid4().hex, + } + } + request = SendStreamingMessageRequest( + id=str(uuid4()), + params=MessageSendParams(**send_message_payload), # type: ignore + ) + + chunks = [] + async for chunk in asend_message_streaming( + request=request, + api_base=None, + litellm_params={ + "custom_llm_provider": "bedrock", + "model": f"bedrock/agentcore/{AGENTCORE_ARN}", + "api_key": "unit-jwt-token", + }, + ): + chunks.append(chunk) + + assert invoke_route.call_count == 1 + invoke_request = invoke_route.calls.last.request + encoded_arn = AGENTCORE_ARN.replace(":", "%3A").replace("/", "%2F") + assert encoded_arn in str(invoke_request.url) + assert invoke_request.headers["authorization"] == "Bearer unit-jwt-token" + + assert len(chunks) == 3 + content = "".join( + chunk["choices"][0]["delta"]["content"] + for chunk in chunks + if chunk["choices"][0].get("delta", {}).get("content") + ) + assert content == "Machine learning." + assert chunks[-1]["choices"][0].get("finish_reason") == "stop" diff --git a/tests/unit/caching/test_disk_cache.py b/tests/unit/caching/test_disk_cache.py index 084370726b1..243f194cc7a 100644 --- a/tests/unit/caching/test_disk_cache.py +++ b/tests/unit/caching/test_disk_cache.py @@ -7,6 +7,7 @@ import pytest pytest.importorskip("diskcache") from litellm.caching.disk_cache import DiskCache +from tests._wait_helpers import await_until class _SlowInt(int): @@ -65,3 +66,93 @@ async def test_async_increment_adds_to_existing_int(cache): async def test_async_increment_treats_non_int_cached_value_as_zero(cache): await cache.async_set_cache("counter", "corrupt") assert await cache.async_increment("counter", 9) == 9 + + +@pytest.fixture +def disk_cache(tmp_path): + import litellm + from litellm.caching import Cache + + litellm.cache = Cache(type="disk", disk_cache_dir=str(tmp_path)) + yield litellm.cache + litellm.cache = None + + +COMPLETION_MESSAGES = [{"role": "user", "content": "write a one sentence poem about: disk-cache-unit-key"}] + + +def test_completion_is_served_from_disk_cache_until_params_change(disk_cache): + import litellm + + first = litellm.completion( + "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, max_tokens=20, mock_response="first" + ) + repeat = litellm.completion( + "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, max_tokens=20, mock_response="second" + ) + changed = litellm.completion( + "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, temperature=0.5, mock_response="third" + ) + + assert (repeat.choices[0].message.content, repeat.id, repeat.created) == ( + first.choices[0].message.content, + first.id, + first.created, + ) + assert changed.choices[0].message.content == "third" + + +async def test_async_completion_is_served_from_disk_cache_until_params_change(disk_cache): + import litellm + + first = await litellm.acompletion( + "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, max_tokens=20, mock_response="first" + ) + await await_until(lambda: len(disk_cache.cache.disk_cache) == 1, message="the first completion is cached") + repeat = await litellm.acompletion( + "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, max_tokens=20, mock_response="second" + ) + changed = await litellm.acompletion( + "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, temperature=0.5, mock_response="third" + ) + + assert (repeat.choices[0].message.content, repeat.id, repeat.created) == ( + first.choices[0].message.content, + first.id, + first.created, + ) + assert changed.choices[0].message.content == "third" + + +async def test_embedding_is_served_from_disk_cache_until_params_change(disk_cache, respx_mock, monkeypatch): + import httpx + + import litellm + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + input_text = ["hello disk-embedding-unit-key"] + embeddings_route = respx_mock.post("https://api.openai.com/v1/embeddings").mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], + "model": "text-embedding-ada-002", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + }, + ) + ) + + await litellm.aembedding("openai/text-embedding-ada-002", input=input_text, caching=True, api_key="sk-unit") + await await_until(lambda: len(disk_cache.cache.disk_cache) == 1, message="the first embedding is cached") + cached = await litellm.aembedding( + "openai/text-embedding-ada-002", input=input_text, caching=True, api_key="sk-unit" + ) + assert cached._hidden_params["cache_hit"] is True + assert cached.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert embeddings_route.call_count == 1 + + await litellm.aembedding( + "openai/text-embedding-ada-002", input=input_text, caching=True, user="charlie", api_key="sk-unit" + ) + assert embeddings_route.call_count == 2 diff --git a/tests/unit/caching/test_gcs_cache.py b/tests/unit/caching/test_gcs_cache.py index fe5df6526be..0887d3d155e 100644 --- a/tests/unit/caching/test_gcs_cache.py +++ b/tests/unit/caching/test_gcs_cache.py @@ -103,3 +103,81 @@ async def test_gcs_cache_async_set_encodes_object_name_in_query(mock_gcs_depende called_url = mock_gcs_dependencies["async_client"].post.call_args.kwargs["url"] assert "name=my_cache%2Fabc123" in called_url + + +@pytest.mark.asyncio +async def test_gcs_cache_completion_hit_and_param_sensitivity(mock_gcs_dependencies): + """Same params hit the injected GCS cache client; different params do not.""" + import asyncio + + import litellm + from litellm.caching import Cache + + store = {} + + import json as _json + from urllib.parse import parse_qs, unquote, urlparse + + def _store(url, data=None, json=None, content=None, **kw): + object_name = parse_qs(urlparse(url).query).get("name", [None])[0] + if object_name is not None: + store[unquote(object_name)] = data if isinstance(data, (str, bytes)) else json + resp = AsyncMock() + resp.status_code = 200 + resp.text = "{}" + return resp + + def _get(url, **kw): + object_name = unquote(url.split("/o/")[-1].split("?")[0]) + resp = AsyncMock() + if object_name in store: + stored = store[object_name] + resp.status_code = 200 + resp.text = stored if isinstance(stored, str) else _json.dumps(stored) + else: + resp.status_code = 404 + resp.text = "not found" + return resp + + mock_gcs_dependencies["async_client"].post.side_effect = _store + mock_gcs_dependencies["async_client"].get.side_effect = _get + mock_gcs_dependencies["sync_client"].post.side_effect = _store + mock_gcs_dependencies["sync_client"].get.side_effect = _get + + messages = [{"role": "user", "content": "write a one sentence poem about: gcs-cache-unit-key"}] + litellm.cache = Cache(type="gcs", gcs_bucket_name="test-bucket") + try: + response1 = await litellm.acompletion( + "gpt-3.5-turbo", + messages=messages, + caching=True, + max_tokens=20, + mock_response="This number is so great!", + ) + deadline = asyncio.get_event_loop().time() + 10 + while not store and asyncio.get_event_loop().time() < deadline: + await asyncio.sleep(0.05) + assert store + response2 = await litellm.acompletion( + "gpt-3.5-turbo", + messages=messages, + caching=True, + max_tokens=20, + mock_response="This number is great!", + ) + assert response1.choices[0].message.content == response2.choices[0].message.content + + response3 = await litellm.acompletion( + "gpt-3.5-turbo", + messages=messages, + caching=True, + temperature=0.5, + mock_response="This number is awful!", + ) + assert response1.choices[0].message.content != response3.choices[0].message.content + finally: + litellm.cache = None + litellm.success_callback = [] + litellm._async_success_callback = [] + + diff --git a/tests/local_testing/create_mock_standard_logging_payload.py b/tests/unit/create_mock_standard_logging_payload.py similarity index 100% rename from tests/local_testing/create_mock_standard_logging_payload.py rename to tests/unit/create_mock_standard_logging_payload.py diff --git a/tests/proxy_behavior/spend/__init__.py b/tests/unit/integrations/clickhouse/__init__.py similarity index 100% rename from tests/proxy_behavior/spend/__init__.py rename to tests/unit/integrations/clickhouse/__init__.py diff --git a/tests/test_litellm/integrations/clickhouse/test_clickhouse_batch_logger.py b/tests/unit/integrations/clickhouse/test_clickhouse_batch_logger.py similarity index 100% rename from tests/test_litellm/integrations/clickhouse/test_clickhouse_batch_logger.py rename to tests/unit/integrations/clickhouse/test_clickhouse_batch_logger.py diff --git a/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py b/tests/unit/integrations/clickhouse/test_clickhouse_spend_logger.py similarity index 95% rename from tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py rename to tests/unit/integrations/clickhouse/test_clickhouse_spend_logger.py index fe47c1a59f3..563937decb9 100644 --- a/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py +++ b/tests/unit/integrations/clickhouse/test_clickhouse_spend_logger.py @@ -20,8 +20,6 @@ from litellm.integrations.clickhouse.clickhouse_spend_logger import ( strip_cache_hit_suffix, ) from litellm.integrations.clickhouse.context import lens_analysis -from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE -from litellm.integrations.custom_batch_logger import CustomBatchLogger from litellm.litellm_core_utils import litellm_logging from litellm.litellm_core_utils.secret_redaction import REDACTED from litellm.tracing.types import SpendLogRecord @@ -151,11 +149,6 @@ def _standard_payload( return payload -def test_is_a_custom_batch_logger(): - assert issubclass(ClickHouseSpendLogger, CustomBatchLogger) - assert ClickHouseSpendLogger.table == SPEND_LOGS_TABLE - - def test_success_row_mapping(): row: Final = spend_log_row_from_payload(cast(StandardLoggingPayload, _payload()), {"response_cost": 0.00042}) @@ -517,25 +510,6 @@ async def test_success_and_failure_events_write_scoped_spend_rows(): assert {key: row[key] for key in original_fields} == original_fields -@pytest.mark.asyncio -async def test_trace_ingest_and_invalid_payload_do_not_write_spend(): - storage = MagicMock() - storage.ensure_schema = AsyncMock() - logger = ClickHouseSpendLogger(storage=storage) - now = datetime.now(timezone.utc) - - await logger.async_log_success_event( - {"standard_logging_object": {**_minimal_payload("trace", status="success", cost=0), "call_type": "/v1/traces"}}, - None, - now, - now, - ) - await logger.async_log_success_event({"standard_logging_object": "invalid"}, None, now, now) - - assert logger.log_queue == [] - storage.ensure_schema.assert_not_awaited() - - @pytest.mark.parametrize( "status,llm_cost,guardrail_cost,expected", [ diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index f8ed11bdca3..7dbb7cc3924 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -10439,7 +10439,7 @@ def test_truncate_standard_logging_payload(): 1. the payload passed in is never modified, since every callback of the request shares it 2. the `messages`, `response`, and `error_str` in the returned payload are truncated """ - from tests.local_testing.create_mock_standard_logging_payload import ( + from tests.unit.create_mock_standard_logging_payload import ( create_standard_logging_payload_with_long_content, ) diff --git a/tests/unit/llms/azure_ai/vector_stores/__init__.py b/tests/unit/llms/azure_ai/vector_stores/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/vector_store_tests/test_azure_ai_vector_store.py b/tests/unit/llms/azure_ai/vector_stores/test_azure_ai_vector_stores_transformation.py similarity index 68% rename from tests/vector_store_tests/test_azure_ai_vector_store.py rename to tests/unit/llms/azure_ai/vector_stores/test_azure_ai_vector_stores_transformation.py index daa0c9b6ed0..7a0df7d96f0 100644 --- a/tests/vector_store_tests/test_azure_ai_vector_store.py +++ b/tests/unit/llms/azure_ai/vector_stores/test_azure_ai_vector_stores_transformation.py @@ -1,5 +1,4 @@ import json -import os from unittest.mock import MagicMock import httpx @@ -19,31 +18,68 @@ from litellm.vector_stores import ( @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio -async def test_basic_search_vector_store(sync_mode): - litellm.turn_on_debug() - litellm.set_verbose = True +async def test_basic_search_vector_store( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, sync_mode +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + query_vector = [0.5, -0.25, 0.125] + search_url = ( + "https://azure-kb-search.search.windows.net/indexes/my-vector-index/docs/search?api-version=2024-07-01" + ) + embedding_route = respx_mock.post(url__regex=r"https://azure-embed\.example\.com/.*/embeddings.*").mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": query_vector}], + "model": "text-embedding-3-large", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + }, + ) + ) + search_route = respx_mock.post(search_url).mock( + return_value=httpx.Response( + 200, + json={ + "value": [ + { + "@search.score": 0.95, + "id": "doc-1", + "content": "azure search document text", + } + ] + }, + ) + ) base_request_args = { "vector_store_id": "my-vector-index", "custom_llm_provider": "azure_ai", "azure_search_service_name": "azure-kb-search", "litellm_embedding_model": "azure_ai/text-embedding-3-large", "litellm_embedding_config": { - "api_base": os.getenv("AZURE_AI_API_BASE"), - "api_key": os.getenv("AZURE_AI_API_KEY"), + "api_base": "https://azure-embed.example.com", + "api_key": "mock_azure_ai_api_key", }, - "api_key": os.getenv("AZURE_SEARCH_API_KEY"), + "api_key": "mock_azure_search_api_key", } - default_query = base_request_args.pop("query", "Basic ping") - print(f"base_request_args: {base_request_args}") - try: - if sync_mode: - response = vector_store_search(query=default_query, **base_request_args) - else: - response = await vector_store_asearch(query=default_query, **base_request_args) - except litellm.InternalServerError: - pytest.skip("Skipping test due to litellm.InternalServerError") + if sync_mode: + response = vector_store_search(query="Basic ping", **base_request_args) + else: + response = await vector_store_asearch(query="Basic ping", **base_request_args) - print("litellm response=", json.dumps(response, indent=4, default=str)) + embedding_request = embedding_route.calls.last.request + assert embedding_request.headers["api-key"] == "mock_azure_ai_api_key" + assert json.loads(embedding_request.read())["input"] == ["Basic ping"] + + search_request = search_route.calls.last.request + assert search_request.headers["api-key"] == "mock_azure_search_api_key" + search_body = json.loads(search_request.read()) + assert search_body["vectorQueries"][0]["vector"] == query_vector + + assert response is not None + assert response["object"] == "vector_store.search_results.page" # type: ignore + assert len(response["data"]) == 1 # type: ignore + assert response["data"][0]["content"][0]["text"] == "azure search document text" # type: ignore class RecordingEmbeddingExecutor: diff --git a/tests/unit/llms/base_llm/vector_store/__init__.py b/tests/unit/llms/base_llm/vector_store/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/base_llm/vector_store/test_base_vector_store.py b/tests/unit/llms/base_llm/vector_store/test_base_vector_store.py new file mode 100644 index 00000000000..c2305861eb2 --- /dev/null +++ b/tests/unit/llms/base_llm/vector_store/test_base_vector_store.py @@ -0,0 +1,98 @@ +"""Cross-provider base vector store contract tests. + +Exercises litellm.vector_stores create/search against a real request/response +cycle with the upstream scripted at the provider's own URL. +""" + +import json +from typing import Final + +import httpx +import pytest +import respx + +import litellm + +OPENAI_VECTOR_STORES_URL: Final = "https://api.openai.com/v1/vector_stores" +OPENAI_VECTOR_STORE_SEARCH_URL: Final = "https://api.openai.com/v1/vector_stores/vs_abc123/search" + + +@pytest.mark.asyncio +async def test_basic_create_vector_store(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + create_route = respx_mock.post(OPENAI_VECTOR_STORES_URL).mock( + return_value=httpx.Response( + 200, + json={ + "id": "vs_abc123", + "object": "vector_store", + "created_at": 1745836841, + "name": "support-docs", + "usage_bytes": 0, + "file_counts": {"in_progress": 0, "completed": 0, "failed": 0, "cancelled": 0, "total": 0}, + "status": "completed", + "last_active_at": 1745836841, + "metadata": {}, + }, + ) + ) + + response = await litellm.vector_stores.acreate( + name="support-docs", + custom_llm_provider="openai", + api_key="sk-unit-test", + ) + + create_request = create_route.calls.last.request + assert create_request.headers["authorization"] == "Bearer sk-unit-test" + create_body = json.loads(create_request.read()) + assert create_body["name"] == "support-docs" + + assert isinstance(response, dict) + assert response["id"] == "vs_abc123" + assert response["object"] == "vector_store" + assert response["name"] == "support-docs" + assert response["created_at"] == 1745836841 + + +@pytest.mark.asyncio +async def test_basic_search_vector_store(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + search_route = respx_mock.post(OPENAI_VECTOR_STORE_SEARCH_URL).mock( + return_value=httpx.Response( + 200, + json={ + "object": "vector_store.search_results.page", + "search_query": "what is machine learning?", + "data": [ + { + "file_id": "file-1", + "filename": "ml-notes.txt", + "score": 0.87, + "attributes": {}, + "content": [{"type": "text", "text": "machine learning answer chunk"}], + } + ], + }, + ) + ) + + response = await litellm.vector_stores.asearch( + query="what is machine learning?", + vector_store_id="vs_abc123", + custom_llm_provider="openai", + api_key="sk-unit-test", + ) + + search_request = search_route.calls.last.request + assert search_request.headers["authorization"] == "Bearer sk-unit-test" + assert json.loads(search_request.read())["query"] == "what is machine learning?" + + assert response["object"] == "vector_store.search_results.page" + assert len(response["data"]) == 1 + first_result = response["data"][0] + assert first_result["file_id"] == "file-1" + assert first_result["filename"] == "ml-notes.txt" + assert first_result["score"] == 0.87 + assert first_result["content"][0]["type"] == "text" + assert first_result["content"][0]["text"] == "machine learning answer chunk" diff --git a/tests/unit/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py b/tests/unit/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py index b45e70e31d3..87c57023c17 100644 --- a/tests/unit/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py +++ b/tests/unit/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py @@ -1,6 +1,11 @@ +import json from typing import Final from unittest.mock import MagicMock +import httpx +import pytest +import respx + from litellm.llms.bedrock.vector_stores.transformation import BedrockVectorStoreConfig @@ -196,3 +201,177 @@ def test_transform_search_request_prefers_extra_body_user_context_over_top_level ) assert body["userContext"] == {"userId": "alice@example.com"} + + +def test_get_file_id_from_metadata(): + """Test that file_id is correctly extracted from metadata.""" + config = BedrockVectorStoreConfig() + + metadata_with_uri = { + "x-amz-bedrock-kb-source-uri": "https://www.litellm.ai", + "x-amz-bedrock-kb-chunk-id": "1%3A0%3AjNYPg5YByRuP5PdK96co", + } + file_id = config._get_file_id_from_metadata(metadata_with_uri) + assert file_id == "https://www.litellm.ai" + + metadata_without_uri = {"x-amz-bedrock-kb-chunk-id": "1%3A0%3AjNYPg5YByRuP5PdK96co"} + file_id = config._get_file_id_from_metadata(metadata_without_uri) + assert file_id == "bedrock-kb-1%3A0%3AjNYPg5YByRuP5PdK96co" + + file_id = config._get_file_id_from_metadata({}) + assert file_id == "bedrock-kb-unknown" + + +def test_get_filename_from_metadata(): + """Test that filename is correctly extracted from metadata.""" + config = BedrockVectorStoreConfig() + + metadata_with_path = {"x-amz-bedrock-kb-source-uri": "https://docs.litellm.ai/tutorial/setup.html"} + filename = config._get_filename_from_metadata(metadata_with_path) + assert filename == "setup.html" + + metadata_domain_only = {"x-amz-bedrock-kb-source-uri": "https://www.litellm.ai"} + filename = config._get_filename_from_metadata(metadata_domain_only) + assert filename == "www.litellm.ai" + + metadata_without_uri = {"x-amz-bedrock-kb-data-source-id": "CCEJIRXXFI"} + filename = config._get_filename_from_metadata(metadata_without_uri) + assert filename == "bedrock-kb-document-CCEJIRXXFI" + + filename = config._get_filename_from_metadata({}) + assert filename == "bedrock-kb-document-unknown" + + +def test_get_attributes_from_metadata(): + """Test that attributes are correctly extracted from metadata.""" + config = BedrockVectorStoreConfig() + + metadata = { + "x-amz-bedrock-kb-source-uri": "https://www.litellm.ai", + "x-amz-bedrock-kb-chunk-id": "1%3A0%3AjNYPg5YByRuP5PdK96co", + "x-amz-bedrock-kb-data-source-id": "CCEJIRXXFI", + } + attributes = config._get_attributes_from_metadata(metadata) + assert attributes == metadata + assert attributes is not metadata + + attributes = config._get_attributes_from_metadata({}) + assert attributes == {} + + attributes = config._get_attributes_from_metadata(None) + assert attributes == {} + + +BEDROCK_RETRIEVE_URL = "https://bedrock-agent-runtime.us-east-1.amazonaws.com/knowledgebases/T37J8R4WTM/retrieve" +BEDROCK_RETRIEVE_RESPONSE = { + "retrievalResults": [ + { + "content": {"text": "Mock search result"}, + "location": {"type": "S3", "s3Location": {"uri": "s3://docs/setup.html"}}, + "metadata": { + "x-amz-bedrock-kb-source-uri": "https://docs.litellm.ai/setup.html", + "x-amz-bedrock-kb-chunk-id": "chunk-1", + }, + "score": 0.9, + } + ] +} + + +@pytest.mark.asyncio +async def test_bedrock_search_with_router(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch): + import litellm + from litellm.router import Router + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + retrieve_route = respx_mock.post(BEDROCK_RETRIEVE_URL).mock( + return_value=httpx.Response(200, json=BEDROCK_RETRIEVE_RESPONSE) + ) + + _router = Router(model_list=[]) + search_response = await _router.avector_store_search( + query="what happens after we add a model", + vector_store_id="T37J8R4WTM", + custom_llm_provider="bedrock", + aws_access_key_id="test_access_key", + aws_secret_access_key="test_secret_key", + aws_region_name="us-east-1", + ) + + assert retrieve_route.call_count == 1 + request = retrieve_route.calls.last.request + assert "AWS4-HMAC-SHA256 Credential=test_access_key" in request.headers["authorization"] + body = json.loads(request.read()) + assert body["retrievalQuery"]["text"] == "what happens after we add a model" + + assert search_response is not None + assert search_response["object"] == "vector_store.search_results.page" + assert len(search_response["data"]) == 1 + assert search_response["data"][0]["score"] == 0.9 + assert search_response["data"][0]["content"][0]["text"] == "Mock search result" + + +@pytest.mark.asyncio +async def test_bedrock_search_with_credentials_managed_registry( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + """ + Test that the vector store search uses the credential accessor from the registry + when AWS environment variables are not set, ensuring credentials are managed properly. + """ + from datetime import datetime, timezone + + import litellm + from litellm.router import Router + from litellm.types.utils import CredentialItem + from litellm.types.vector_stores import LiteLLM_ManagedVectorStore + from litellm.vector_stores.vector_store_registry import VectorStoreRegistry + + test_credentials = CredentialItem( + credential_name="bedrock-litellm-website-knowledgebase", + credential_info={ + "provider": "aws", + "description": "Test AWS credentials for bedrock", + }, + credential_values={ + "aws_access_key_id": "test_access_key", + "aws_secret_access_key": "test_secret_key", + "aws_region_name": "us-east-1", + }, + ) + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "credential_list", [test_credentials]) + monkeypatch.delenv("AWS_ACCESS_KEY_ID", raising=False) + monkeypatch.delenv("AWS_SECRET_ACCESS_KEY", raising=False) + monkeypatch.delenv("AWS_REGION_NAME", raising=False) + + vector_store = LiteLLM_ManagedVectorStore( + vector_store_id="T37J8R4WTM", + custom_llm_provider="bedrock", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + litellm_credential_name="bedrock-litellm-website-knowledgebase", + ) + + registry = VectorStoreRegistry([vector_store]) + monkeypatch.setattr(litellm, "vector_store_registry", registry) + + retrieve_route = respx_mock.post(BEDROCK_RETRIEVE_URL).mock( + return_value=httpx.Response(200, json=BEDROCK_RETRIEVE_RESPONSE) + ) + + _router = Router(model_list=[]) + search_response = await _router.avector_store_search( + query="what happens after we add a model", + vector_store_id="T37J8R4WTM", + custom_llm_provider="bedrock", + ) + + assert retrieve_route.call_count == 1 + request = retrieve_route.calls.last.request + assert "AWS4-HMAC-SHA256 Credential=test_access_key" in request.headers["authorization"] + + assert search_response["data"][0]["score"] == 0.9 + assert search_response["data"][0]["content"][0]["text"] == "Mock search result" diff --git a/tests/unit/llms/milvus/__init__.py b/tests/unit/llms/milvus/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/milvus/vector_stores/__init__.py b/tests/unit/llms/milvus/vector_stores/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/milvus/vector_stores/test_milvus_vector_stores_transformation.py b/tests/unit/llms/milvus/vector_stores/test_milvus_vector_stores_transformation.py new file mode 100644 index 00000000000..01da16a6efd --- /dev/null +++ b/tests/unit/llms/milvus/vector_stores/test_milvus_vector_stores_transformation.py @@ -0,0 +1,476 @@ +""" +Tests for Milvus Vector Store +""" + +import json +from unittest.mock import MagicMock + +import httpx +import pytest +import respx + +import litellm +from litellm import Router +from litellm.llms.milvus.vector_stores.transformation import MilvusVectorStoreConfig +from litellm.types.utils import EmbeddingResponse +from litellm.vector_stores import asearch as vector_store_asearch +from litellm.vector_stores import search as vector_store_search + +MOCK_MILVUS_SEARCH_RESPONSE = { + "code": 0, + "cost": 6, + "data": [ + { + "book_id": 0, + "book_intro_text": "abababababa_0562efee-0f1f-4b6b-9ca3-1a160f124ad8", + "distance": 10.240219, + }, + { + "book_id": 1, + "book_intro_text": "abababababa_9a13e8f3-bb1e-487f-b555-b8ae4b127243", + "distance": 10.240219, + }, + { + "book_id": 2, + "book_intro_text": "abababababa_870f47f1-23ec-4364-ad30-6d364ba8ddb5", + "distance": 10.240219, + }, + { + "book_id": 1000, + "book_intro_text": "abababababa_8ea2d76a-3fdf-49b3-8f16-a91638361bba", + "distance": 8.531628, + }, + { + "book_id": 1001, + "book_intro_text": "abababababa_24758251-e740-4183-8649-2f742f676ca0", + "distance": 8.531628, + }, + { + "book_id": 1002, + "book_intro_text": "abababababa_faa55789-220d-4ef1-b5bf-a72f2fbd061b", + "distance": 8.531628, + }, + { + "book_id": 0, + "book_intro_text": "abababababa_0562efee-0f1f-4b6b-9ca3-1a160f124ad8", + "distance": 8.236887, + }, + { + "book_id": 1, + "book_intro_text": "abababababa_9a13e8f3-bb1e-487f-b555-b8ae4b127243", + "distance": 8.236887, + }, + { + "book_id": 2, + "book_intro_text": "abababababa_870f47f1-23ec-4364-ad30-6d364ba8ddb5", + "distance": 8.236887, + }, + ], + "topks": [3, 3, 3], +} + + +class RecordingEmbeddingExecutor: + def __init__(self, response): + self.response = response + self.calls = [] + + def embed(self, model, query, configuration): + self.calls.append((model, query, dict(configuration))) + return self.response + + async def aembed(self, model, query, configuration): + self.calls.append((model, query, dict(configuration))) + return self.response + + +ALIAS_QUERY_VECTOR = [0.5, -0.25, 0.125] +ALIAS_EMBEDDING_RESPONSE = EmbeddingResponse( + data=[{"embedding": ALIAS_QUERY_VECTOR, "index": 0, "object": "embedding"}] +) +OPENAI_EMBEDDINGS_URL = "https://api.openai.com/v1/embeddings" +MILVUS_SEARCH_URL = "https://milvus.example/v2/vectordb/entities/search" +ZILLIZ_SEARCH_URL = ( + "https://in03-test.serverless.aws-eu-central-1.cloud.zilliz.com/v2/vectordb/entities/search" +) +ALIAS_SEARCH_KWARGS = { + "query": "what is machine learning?", + "vector_store_id": "book_2", + "custom_llm_provider": "milvus", + "api_base": "https://milvus.example", + "api_key": "mock_milvus_api_key", + "litellm_embedding_model": "multilingual-e5-large", + "milvus_text_field": "book_intro_text", +} +ZILLIZ_SEARCH_KWARGS = { + "query": "what is machine learning?", + "vector_store_id": "book_2", + "custom_llm_provider": "milvus", + "api_base": "https://in03-test.serverless.aws-eu-central-1.cloud.zilliz.com", + "api_key": "mock_milvus_api_key", + "litellm_embedding_model": "text-embedding-3-large", + "litellm_embedding_config": { + "api_key": "mock_openai_api_key", + }, + "outputFields": ["book_intro_text"], + "annsField": "book_intro_vector", + "milvus_text_field": "book_intro_text", +} + + +def _alias_router(): + return Router( + model_list=[ + { + "model_name": "multilingual-e5-large", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_key": "deployment-key", + }, + } + ] + ) + + +def _mock_embedding_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(OPENAI_EMBEDDINGS_URL).mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": ALIAS_QUERY_VECTOR}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + }, + ) + ) + + +def _mock_search_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(MILVUS_SEARCH_URL).mock(return_value=httpx.Response(200, json=MOCK_MILVUS_SEARCH_RESPONSE)) + + +def _mock_zilliz_search_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(ZILLIZ_SEARCH_URL).mock(return_value=httpx.Response(200, json=MOCK_MILVUS_SEARCH_RESPONSE)) + + +def _search_request_body(search_route: respx.Route): + return json.loads(search_route.calls.last.request.read()) + + +class TestMilvusVectorStore: + """Test Milvus Vector Store with mocked responses""" + + @pytest.mark.asyncio + async def test_basic_search_with_mock_async(self, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch): + """Test basic vector search with mocked backend response (async)""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + embedding_route = _mock_embedding_route(respx_mock) + search_route = _mock_zilliz_search_route(respx_mock) + + response = await vector_store_asearch(**ZILLIZ_SEARCH_KWARGS) + + embedding_request = embedding_route.calls.last.request + assert embedding_request.headers["authorization"] == "Bearer mock_openai_api_key" + embedding_body = json.loads(embedding_request.read()) + assert embedding_body["model"] == "text-embedding-3-large" + assert embedding_body["input"] == ["what is machine learning?"] + + search_request = search_route.calls.last.request + request_data = json.loads(search_request.read()) + assert request_data["collectionName"] == "book_2" + assert request_data["data"] == [ALIAS_QUERY_VECTOR] + assert request_data["annsField"] == "book_intro_vector" + assert request_data["outputFields"] == ["book_intro_text"] + assert "v2/vectordb/entities/search" in str(search_request.url) + + assert response is not None + assert response["object"] == "vector_store.search_results.page" # type: ignore + assert len(response["data"]) == 9 # type: ignore + + first_result = response["data"][0] # type: ignore + assert first_result["score"] == 10.240219 # type: ignore + assert len(first_result["content"]) > 0 # type: ignore + assert first_result["content"][0]["type"] == "text" # type: ignore + assert first_result["content"][0]["text"] == "abababababa_0562efee-0f1f-4b6b-9ca3-1a160f124ad8" # type: ignore + assert first_result["attributes"]["book_id"] == 0 # type: ignore + assert "book_intro_text" not in first_result["attributes"] # type: ignore + + def test_basic_search_with_mock_sync(self, respx_mock: respx.MockRouter): + """Test basic vector search with mocked backend response (sync)""" + embedding_route = _mock_embedding_route(respx_mock) + search_route = _mock_zilliz_search_route(respx_mock) + + response = vector_store_search(**ZILLIZ_SEARCH_KWARGS) + + assert embedding_route.call_count == 1 + assert search_route.call_count == 1 + + request_data = _search_request_body(search_route) + assert request_data["collectionName"] == "book_2" + assert request_data["data"] == [ALIAS_QUERY_VECTOR] + assert request_data["annsField"] == "book_intro_vector" + assert request_data["outputFields"] == ["book_intro_text"] + + assert response is not None + assert response["object"] == "vector_store.search_results.page" # type: ignore + assert len(response["data"]) == 9 # type: ignore + assert "search_query" in response # type: ignore + + expected_results = [ + { + "book_id": 0, + "text": "abababababa_0562efee-0f1f-4b6b-9ca3-1a160f124ad8", + "distance": 10.240219, + }, + { + "book_id": 1, + "text": "abababababa_9a13e8f3-bb1e-487f-b555-b8ae4b127243", + "distance": 10.240219, + }, + { + "book_id": 2, + "text": "abababababa_870f47f1-23ec-4364-ad30-6d364ba8ddb5", + "distance": 10.240219, + }, + ] + + for idx, expected in enumerate(expected_results): + result = response["data"][idx] # type: ignore + assert result["score"] == expected["distance"] # type: ignore + assert len(result["content"]) > 0 # type: ignore + assert result["content"][0]["type"] == "text" # type: ignore + assert result["content"][0]["text"] == expected["text"] # type: ignore + assert result["attributes"]["book_id"] == expected["book_id"] # type: ignore + assert "book_intro_text" not in result["attributes"] # type: ignore + + def test_user_supplied_db_and_partition_are_dropped(self, respx_mock: respx.MockRouter): + """User-supplied dbName / partitionNames must not be forwarded to Milvus.""" + _mock_embedding_route(respx_mock) + search_route = _mock_zilliz_search_route(respx_mock) + + vector_store_search( + **ZILLIZ_SEARCH_KWARGS, + dbName="other_tenant_db", + partitionNames=["other_tenant_partition"], + ) + + request_data = _search_request_body(search_route) + assert "dbName" not in request_data + assert "partitionNames" not in request_data + assert request_data["collectionName"] == "book_2" + assert request_data["annsField"] == "book_intro_vector" + assert request_data["outputFields"] == ["book_intro_text"] + + def test_backend_configured_db_and_partition_are_forwarded(self, respx_mock: respx.MockRouter): + """milvus_db_name / milvus_partition_names from litellm_params must be sent.""" + _mock_embedding_route(respx_mock) + search_route = _mock_zilliz_search_route(respx_mock) + + vector_store_search( + **ZILLIZ_SEARCH_KWARGS, + milvus_db_name="tenant_a_db", + milvus_partition_names=["tenant_a_partition"], + ) + + request_data = _search_request_body(search_route) + assert request_data["dbName"] == "tenant_a_db" + assert request_data["partitionNames"] == ["tenant_a_partition"] + + def test_user_params_cannot_override_backend_db_and_partition(self, respx_mock: respx.MockRouter): + """Backend-config dbName/partitionNames must win over user-supplied values.""" + _mock_embedding_route(respx_mock) + search_route = _mock_zilliz_search_route(respx_mock) + + vector_store_search( + **ZILLIZ_SEARCH_KWARGS, + milvus_db_name="tenant_a_db", + milvus_partition_names=["tenant_a_partition"], + dbName="other_tenant_db", + partitionNames=["other_tenant_partition"], + ) + + request_data = _search_request_body(search_route) + assert request_data["dbName"] == "tenant_a_db" + assert request_data["partitionNames"] == ["tenant_a_partition"] + + +def _assert_alias_resolved(embedding_route: respx.Route, search_route: respx.Route, response): + embedding_request = embedding_route.calls.last.request + assert embedding_request.headers["authorization"] == "Bearer deployment-key" + embedding_body = json.loads(embedding_request.read()) + assert embedding_body["model"] == "text-embedding-3-small" + assert embedding_body["input"] == ["what is machine learning?"] + search_request = search_route.calls.last.request + assert search_request.headers["authorization"] == "Bearer mock_milvus_api_key" + assert json.loads(search_request.read())["data"] == [ALIAS_QUERY_VECTOR] + assert len(response["data"]) == len(MOCK_MILVUS_SEARCH_RESPONSE["data"]) + assert response["data"][0]["content"][0]["text"] == MOCK_MILVUS_SEARCH_RESPONSE["data"][0]["book_intro_text"] + + +def test_router_search_resolves_bare_embedding_alias_sync( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + embedding_route = _mock_embedding_route(respx_mock) + search_route = _mock_search_route(respx_mock) + + response = _alias_router().vector_store_search(**ALIAS_SEARCH_KWARGS) + + _assert_alias_resolved(embedding_route, search_route, response) + + +@pytest.mark.asyncio +async def test_router_search_resolves_bare_embedding_alias_async( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + embedding_route = _mock_embedding_route(respx_mock) + search_route = _mock_search_route(respx_mock) + + response = await _alias_router().avector_store_search(**ALIAS_SEARCH_KWARGS) + + _assert_alias_resolved(embedding_route, search_route, response) + + +def test_sdk_search_with_router_kwarg_resolves_bare_embedding_alias_sync( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + embedding_route = _mock_embedding_route(respx_mock) + search_route = _mock_search_route(respx_mock) + + response = litellm.vector_stores.search(router=_alias_router(), **ALIAS_SEARCH_KWARGS) + + _assert_alias_resolved(embedding_route, search_route, response) + + +@pytest.mark.asyncio +async def test_sdk_search_with_router_kwarg_resolves_bare_embedding_alias_async( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + embedding_route = _mock_embedding_route(respx_mock) + search_route = _mock_search_route(respx_mock) + + response = await litellm.vector_stores.asearch(router=_alias_router(), **ALIAS_SEARCH_KWARGS) + + _assert_alias_resolved(embedding_route, search_route, response) + + +def _team_alias_router(): + return Router( + model_list=[ + { + "model_name": "team-a-embedder", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_key": "deployment-key", + }, + "model_info": {"team_id": "team-a", "team_public_model_name": "multilingual-e5-large"}, + } + ] + ) + + +@pytest.mark.asyncio +async def test_sdk_search_with_router_kwarg_resolves_team_alias_from_request_metadata( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + embedding_route = _mock_embedding_route(respx_mock) + search_route = _mock_search_route(respx_mock) + + response = await litellm.vector_stores.asearch( + router=_team_alias_router(), metadata={"user_api_key_team_id": "team-a"}, **ALIAS_SEARCH_KWARGS + ) + + _assert_alias_resolved(embedding_route, search_route, response) + + +@pytest.mark.asyncio +async def test_sdk_search_with_router_kwarg_rejects_team_alias_without_team_metadata( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + embedding_route = _mock_embedding_route(respx_mock) + _mock_search_route(respx_mock) + + with pytest.raises(litellm.APIConnectionError): + await litellm.vector_stores.asearch(router=_team_alias_router(), **ALIAS_SEARCH_KWARGS) + + assert embedding_route.call_count == 0 + + +@pytest.mark.asyncio +async def test_transform_uses_injected_executor_without_embedding_config(respx_mock: respx.MockRouter): + executor = RecordingEmbeddingExecutor(ALIAS_EMBEDDING_RESPONSE) + config = MilvusVectorStoreConfig() + logging_obj = MagicMock() + logging_obj.model_call_details = {} + transform_kwargs = { + "vector_store_id": "book_2", + "query": ["what is", "milvus?"], + "vector_store_search_optional_params": {"limit": 3}, + "api_base": "https://milvus.example", + "litellm_logging_obj": logging_obj, + "litellm_params": {"litellm_embedding_model": "multilingual-e5-large", "milvus_db_name": "docs"}, + "embedding_executor": executor, + } + + url, sync_body = config.transform_search_vector_store_request(**transform_kwargs) + _, async_body = await config.atransform_search_vector_store_request(**transform_kwargs) + + assert respx_mock.calls.call_count == 0 + assert executor.calls == [("multilingual-e5-large", "what is milvus?", {})] * 2 + assert url == MILVUS_SEARCH_URL + assert sync_body == async_body + assert sync_body == { + "collectionName": "book_2", + "data": [ALIAS_QUERY_VECTOR], + "annsField": "book_intro_vector", + "limit": 3, + "dbName": "docs", + } + assert logging_obj.model_call_details["input"] == "what is milvus?" + assert logging_obj.model_call_details["embedding_model"] == "multilingual-e5-large" + + +def test_transform_falls_back_to_sdk_embedding_without_executor_or_config( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setenv("OPENAI_API_KEY", "env-key") + embedding_route = _mock_embedding_route(respx_mock) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + + _, body = MilvusVectorStoreConfig().transform_search_vector_store_request( + vector_store_id="book_2", + query="q", + vector_store_search_optional_params={}, + api_base="https://milvus.example", + litellm_logging_obj=logging_obj, + litellm_params={"litellm_embedding_model": "openai/text-embedding-3-small"}, + ) + + embedding_request = embedding_route.calls.last.request + assert embedding_request.headers["authorization"] == "Bearer env-key" + assert json.loads(embedding_request.read())["input"] == ["q"] + assert body["data"] == [ALIAS_QUERY_VECTOR] + + +def test_transform_requires_embedding_model(): + with pytest.raises(ValueError, match="litellm_embedding_model is required"): + MilvusVectorStoreConfig().transform_search_vector_store_request( + vector_store_id="book_2", + query="q", + vector_store_search_optional_params={}, + api_base="https://milvus.example", + litellm_logging_obj=MagicMock(), + litellm_params={"litellm_embedding_config": {"api_key": "store-key"}}, + embedding_executor=RecordingEmbeddingExecutor(ALIAS_EMBEDDING_RESPONSE), + ) diff --git a/tests/unit/llms/ragflow/vector_stores/__init__.py b/tests/unit/llms/ragflow/vector_stores/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/ragflow/vector_stores/test_ragflow_vector_stores_transformation.py b/tests/unit/llms/ragflow/vector_stores/test_ragflow_vector_stores_transformation.py new file mode 100644 index 00000000000..f1fc34100d1 --- /dev/null +++ b/tests/unit/llms/ragflow/vector_stores/test_ragflow_vector_stores_transformation.py @@ -0,0 +1,253 @@ +""" +Test RAGFlow Vector Store helper functions and transformation. +""" + +import os +from unittest.mock import MagicMock, Mock, patch + +import httpx +import pytest + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.ragflow.vector_stores.transformation import RAGFlowVectorStoreConfig +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.types.vector_stores import VectorStoreCreateOptionalRequestParams + + +def test_get_auth_credentials(): + """Test that auth credentials are correctly extracted.""" + config = RAGFlowVectorStoreConfig() + + litellm_params = {"api_key": "test-api-key-123"} + credentials = config.get_auth_credentials(litellm_params) + assert "headers" in credentials + assert credentials["headers"]["Authorization"] == "Bearer test-api-key-123" + + with pytest.raises(ValueError, match="api_key is required"): + config.get_auth_credentials({}) + + +def test_get_complete_url(): + """Test that complete URL is correctly constructed.""" + config = RAGFlowVectorStoreConfig() + + litellm_params = {"api_base": "http://custom-host:9999"} + url = config.get_complete_url(api_base=None, litellm_params=litellm_params) + assert url == "http://custom-host:9999/api/v1/datasets" + + url = config.get_complete_url(api_base="http://test-host:8888", litellm_params={}) + assert url == "http://test-host:8888/api/v1/datasets" + + with patch.dict(os.environ, {}, clear=True): + url = config.get_complete_url(api_base=None, litellm_params={}) + assert url == "http://localhost:9380/api/v1/datasets" + + url = config.get_complete_url(api_base="http://test-host:8888/", litellm_params={}) + assert url == "http://test-host:8888/api/v1/datasets" + + +def test_validate_environment(): + """Test environment validation and header setting.""" + config = RAGFlowVectorStoreConfig() + from litellm.types.router import GenericLiteLLMParams + + litellm_params = GenericLiteLLMParams(api_key="test-key") + headers = config.validate_environment({}, litellm_params) + assert headers["Authorization"] == "Bearer test-key" + assert headers["Content-Type"] == "application/json" + + with pytest.raises(ValueError, match="RAGFLOW_API_KEY"): + config.validate_environment({}, GenericLiteLLMParams()) + + +def test_get_vector_store_endpoints_by_type(): + """Test that endpoints are correctly configured (empty for management only).""" + config = RAGFlowVectorStoreConfig() + endpoints = config.get_vector_store_endpoints_by_type() + assert endpoints["read"] == [] + assert endpoints["write"] == [] + + +def test_transform_create_vector_store_request_basic(): + """Test basic dataset creation request transformation.""" + config = RAGFlowVectorStoreConfig() + + params: VectorStoreCreateOptionalRequestParams = {"name": "test-dataset"} + + url, body = config.transform_create_vector_store_request( + params, "http://localhost:9380/api/v1/datasets" + ) + + assert url == "http://localhost:9380/api/v1/datasets" + assert body["name"] == "test-dataset" + assert body["chunk_method"] == "naive" + + +def test_transform_create_vector_store_request_with_metadata(): + """Test dataset creation with RAGFlow-specific metadata.""" + config = RAGFlowVectorStoreConfig() + + params: VectorStoreCreateOptionalRequestParams = { + "name": "test-dataset-advanced", + "metadata": { + "description": "Test dataset", + "embedding_model": "BAAI/bge-large-zh-v1.5@BAAI", + "permission": "me", + "chunk_method": "naive", + "parser_config": {"chunk_token_num": 512, "delimiter": "\n"}, + }, + } + + url, body = config.transform_create_vector_store_request( + params, "http://localhost:9380/api/v1/datasets" + ) + + assert body["name"] == "test-dataset-advanced" + assert body["description"] == "Test dataset" + assert body["embedding_model"] == "BAAI/bge-large-zh-v1.5@BAAI" + assert body["permission"] == "me" + assert body["chunk_method"] == "naive" + assert "parser_config" in body + assert body["parser_config"]["chunk_token_num"] == 512 + + +def test_transform_create_vector_store_request_missing_name(): + """Test that missing name raises ValueError.""" + config = RAGFlowVectorStoreConfig() + + params: VectorStoreCreateOptionalRequestParams = {} + + with pytest.raises(ValueError, match="name is required"): + config.transform_create_vector_store_request( + params, "http://localhost:9380/api/v1/datasets" + ) + + +def test_transform_create_vector_store_request_mutually_exclusive(): + """Test that chunk_method and pipeline_id are mutually exclusive.""" + config = RAGFlowVectorStoreConfig() + + params: VectorStoreCreateOptionalRequestParams = { + "name": "test-dataset", + "metadata": { + "chunk_method": "naive", + "pipeline_id": "d0bebe30ae2211f0970942010a8e0005", + }, + } + + with pytest.raises(ValueError, match="mutually exclusive"): + config.transform_create_vector_store_request( + params, "http://localhost:9380/api/v1/datasets" + ) + + +def test_transform_create_vector_store_request_with_pipeline(): + """Test dataset creation with ingestion pipeline.""" + config = RAGFlowVectorStoreConfig() + + params: VectorStoreCreateOptionalRequestParams = { + "name": "test-pipeline-dataset", + "metadata": { + "parse_type": 2, + "pipeline_id": "d0bebe30ae2211f0970942010a8e0005", + }, + } + + url, body = config.transform_create_vector_store_request( + params, "http://localhost:9380/api/v1/datasets" + ) + + assert body["name"] == "test-pipeline-dataset" + assert body["parse_type"] == 2 + assert body["pipeline_id"] == "d0bebe30ae2211f0970942010a8e0005" + assert "chunk_method" not in body + + +def test_transform_create_vector_store_response_success(): + """Test successful response transformation.""" + config = RAGFlowVectorStoreConfig() + + mock_response = Mock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {} + mock_response.json.return_value = { + "code": 0, + "data": { + "id": "3b4de7d4241d11f0a6a79f24fc270c7f", + "name": "test-dataset", + "create_time": 1745836841611, + "chunk_method": "naive", + "embedding_model": "BAAI/bge-large-zh-v1.5@BAAI", + }, + } + + response = config.transform_create_vector_store_response(mock_response) + + assert response["id"] == "3b4de7d4241d11f0a6a79f24fc270c7f" + assert response["name"] == "test-dataset" + assert response["object"] == "vector_store" + assert response["status"] == "completed" + assert response["created_at"] == 1745836841 + assert response["bytes"] == 0 + assert "file_counts" in response + + +def test_transform_create_vector_store_response_error(): + """Test error response transformation.""" + config = RAGFlowVectorStoreConfig() + + mock_response = Mock(spec=httpx.Response) + mock_response.status_code = 400 + mock_response.headers = {} + mock_response.json.return_value = { + "code": 101, + "message": "Dataset name 'test-dataset' already exists", + } + + with pytest.raises(BaseLLMException): + config.transform_create_vector_store_response(mock_response) + + +def test_transform_create_vector_store_response_missing_id(): + """Test response with missing dataset ID.""" + config = RAGFlowVectorStoreConfig() + + mock_response = Mock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {} + mock_response.json.return_value = { + "code": 0, + "data": { + "name": "test-dataset" + }, + } + + with pytest.raises(ValueError, match="missing dataset id"): + config.transform_create_vector_store_response(mock_response) + + +def test_transform_search_vector_store_request_not_implemented(): + """Test that search operations raise NotImplementedError.""" + config = RAGFlowVectorStoreConfig() + logging_obj = MagicMock(spec=LiteLLMLoggingObj) + + with pytest.raises(NotImplementedError, match="management only"): + config.transform_search_vector_store_request( + vector_store_id="test-id", + query="test query", + vector_store_search_optional_params={}, + api_base="http://localhost:9380", + litellm_logging_obj=logging_obj, + litellm_params={}, + extra_body=None, + ) + + +def test_transform_search_vector_store_response_not_implemented(): + """Test that search response transformation raises NotImplementedError.""" + config = RAGFlowVectorStoreConfig() + logging_obj = MagicMock(spec=LiteLLMLoggingObj) + mock_response = Mock(spec=httpx.Response) + + with pytest.raises(NotImplementedError, match="management only"): + config.transform_search_vector_store_response(mock_response, logging_obj) diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_search_vector_store_transformation.py b/tests/unit/llms/vertex_ai/test_vertex_ai_search_vector_store_transformation.py index f3a276f7e46..256bebc1c75 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_ai_search_vector_store_transformation.py +++ b/tests/unit/llms/vertex_ai/test_vertex_ai_search_vector_store_transformation.py @@ -1,7 +1,11 @@ +import json from collections.abc import Mapping from types import SimpleNamespace +from typing import Final +import httpx import pytest +import respx from litellm.exceptions import BadRequestError from litellm.llms.vertex_ai.vector_stores.search_api.transformation import ( @@ -484,3 +488,138 @@ def test_search_response_keeps_link_metadata_and_positional_scores(): def test_search_response_without_results_key_is_empty(): assert _search_response({"totalSize": 0})["data"] == [] + + +MOCK_VERTEX_SEARCH_RESPONSE: Final = { + "results": [ + { + "id": "0", + "document": { + "name": "projects/648660250433/locations/global/collections/default_collection/dataStores/litellm-docs_1761094140318/branches/0/documents/0", + "id": "0", + "derivedStructData": { + "htmlTitle": "LiteLLM - Getting Started | liteLLM", + "snippets": [ + { + "htmlSnippet": "https://github.com/BerriAI/litellm.", + "snippet": "https://github.com/BerriAI/litellm.", + } + ], + "title": "LiteLLM - Getting Started | liteLLM", + "link": "https://docs.litellm.ai/docs/", + "displayLink": "docs.litellm.ai", + }, + }, + }, + { + "id": "1", + "document": { + "name": "projects/648660250433/locations/global/collections/default_collection/dataStores/litellm-docs_1761094140318/branches/0/documents/1", + "id": "1", + "derivedStructData": { + "title": "Using Vector Stores (Knowledge Bases) | liteLLM", + "link": "https://docs.litellm.ai/docs/completion/knowledgebase", + "snippets": [ + { + "snippet": "LiteLLM integrates with vector stores, allowing your models to access your organization's data." + } + ], + }, + }, + }, + ], + "totalSize": 299, + "attributionToken": "mock_token", + "summary": {}, +} +GOOGLE_TOKEN_URL: Final = "https://oauth2.googleapis.com/token" +VERTEX_AUTHORIZED_USER_CREDENTIALS: Final = json.dumps( + { + "type": "authorized_user", + "client_id": "unit-client-id", + "client_secret": "unit-client-secret", + "refresh_token": "unit-refresh-token", + "quota_project_id": "test-vector-store-db", + } +) + + +def _mock_google_token_refresh(requests_mock) -> None: + requests_mock.post( + GOOGLE_TOKEN_URL, + json={"access_token": "ya29.unit-token", "expires_in": 3600, "token_type": "Bearer"}, + ) + + +def _mock_vertex_search(respx_mock) -> "respx.Route": + return respx_mock.post(url__regex=r"https://discoveryengine\.googleapis\.com/.*:search.*").mock( + return_value=httpx.Response(200, json=MOCK_VERTEX_SEARCH_RESPONSE) + ) + + +@pytest.mark.asyncio +async def test_basic_search_with_mock(respx_mock, requests_mock, monkeypatch: pytest.MonkeyPatch): + """Test basic vector search with mocked backend response""" + import litellm + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + _mock_google_token_refresh(requests_mock) + search_route = _mock_vertex_search(respx_mock) + + response = await litellm.vector_stores.asearch( + query="what is LiteLLM?", + vector_store_id="test-litellm-app_1761094730750", + custom_llm_provider="vertex_ai/search_api", + vertex_project="test-vector-store-db", + vertex_location="us-central1", + vertex_credentials=VERTEX_AUTHORIZED_USER_CREDENTIALS, + ) + + assert response is not None + assert response["object"] == "vector_store.search_results.page" + assert len(response["data"]) > 0 + assert "search_query" in response + + first_result = response["data"][0] + assert len(first_result["content"]) > 0 + assert first_result["content"][0]["type"] == "text" + assert "text" in first_result["content"][0] + + search_request = search_route.calls.last.request + url = str(search_request.url) + assert "discoveryengine.googleapis.com" in url + assert "test-vector-store-db" in url + assert "test-litellm-app_1761094730750" in url + assert search_request.headers["authorization"] == "Bearer ya29.unit-token" + + +def test_basic_search_sync_with_mock(respx_mock, requests_mock): + """Test basic vector search (sync) with mocked backend response""" + _mock_google_token_refresh(requests_mock) + search_route = _mock_vertex_search(respx_mock) + + response = __import__("litellm").vector_stores.search( + query="what is LiteLLM?", + vector_store_id="test-litellm-app_1761094730750", + custom_llm_provider="vertex_ai/search_api", + vertex_project="test-vector-store-db", + vertex_location="us-central1", + vertex_credentials=VERTEX_AUTHORIZED_USER_CREDENTIALS, + ) + + assert response is not None + assert response["object"] == "vector_store.search_results.page" + assert len(response["data"]) > 0 + assert "search_query" in response + + first_result = response["data"][0] + assert len(first_result["content"]) > 0 + assert first_result["content"][0]["type"] == "text" + assert "text" in first_result["content"][0] + + assert "document_id" in first_result["attributes"] + assert "link" in first_result["attributes"] + assert "title" in first_result["attributes"] + + assert search_route.call_count == 1 + assert search_route.calls.last.request.headers["authorization"] == "Bearer ya29.unit-token" diff --git a/tests/unit/proxy/analytics_endpoints/test_analytics_endpoints.py b/tests/unit/proxy/analytics_endpoints/test_analytics_endpoints.py index ea528878db8..69188ebd5a2 100644 --- a/tests/unit/proxy/analytics_endpoints/test_analytics_endpoints.py +++ b/tests/unit/proxy/analytics_endpoints/test_analytics_endpoints.py @@ -119,7 +119,7 @@ async def test_filters_are_passed_to_sql_as_json_arrays(mock_prisma: MagicMock): async def test_every_query_excludes_the_same_info_routes(mock_prisma: MagicMock): """Regression for LIT-5884: failed info-route calls are spend-logged but are not inference traffic, so the groups, error breakdown and both filter-option queries all receive the same exclusion list. What - the SQL does with it is covered against Postgres in tests/proxy_behavior/spend/test_cache_activity.py.""" + the SQL does with it is covered against Postgres in tests/integration/spend/test_cache_activity.py.""" await get_global_activity(start_date="2026-07-01", end_date="2026-07-27", key_aliases=[], models=[]) exclusions_by_query = {call.args[0]: json.loads(call.args[-1]) for call in mock_prisma.db.query_raw.call_args_list} diff --git a/tests/unit/proxy/db/test_autorouter_session_rollup.py b/tests/unit/proxy/db/test_autorouter_session_rollup.py index 686c5a3a970..4217feeb83b 100644 --- a/tests/unit/proxy/db/test_autorouter_session_rollup.py +++ b/tests/unit/proxy/db/test_autorouter_session_rollup.py @@ -2,7 +2,7 @@ Unit tests for the auto-router per-session benchmarks rollup writer. The classification SQL itself runs against a real Postgres in -tests/proxy_behavior/spend/test_autorouter_session_rollup.py; these tests cover the +tests/integration/spend/test_autorouter_session_rollup.py; these tests cover the request-time transaction builder and the flush contract with an injected fake client. """ diff --git a/tests/unit/proxy/management_endpoints/test_encryption_endpoints.py b/tests/unit/proxy/management_endpoints/test_encryption_endpoints.py index e92cb3b13a7..9398bd795fc 100644 --- a/tests/unit/proxy/management_endpoints/test_encryption_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_encryption_endpoints.py @@ -3,7 +3,7 @@ The endpoint bodies are exercised directly with the migration engine mocked, so the admin guard, db-not-connected guard, and success path are all covered without touching a live DB. The live ASGI/auth contract is covered separately in -``tests/proxy_behavior/management/test_credential_migration_endpoint.py``. +``tests/integration/management/test_credential_migration_endpoint.py``. """ from types import SimpleNamespace diff --git a/tests/unit/proxy/management_endpoints/test_management_route_coverage.py b/tests/unit/proxy/management_endpoints/test_management_route_coverage.py new file mode 100644 index 00000000000..64e584b4f70 --- /dev/null +++ b/tests/unit/proxy/management_endpoints/test_management_route_coverage.py @@ -0,0 +1,88 @@ +import ast +from collections.abc import Iterator +from itertools import chain +from pathlib import Path +import re +from typing import Final + +REPO_ROOT: Final = Path(__file__).resolve().parents[4] +SOURCE_FILES: Final = ( + REPO_ROOT / "litellm/proxy/management_endpoints/key_management_endpoints.py", + REPO_ROOT / "litellm/proxy/management_endpoints/team_endpoints.py", +) +TEST_DIRS: Final = ( + REPO_ROOT / "tests/integration/management", + REPO_ROOT / "tests/integration/authorization", +) +SELF: Final = Path(__file__).resolve() + +_ROUTE_DECORATOR = re.compile( + r"@router\.(?:get|post|put|delete|patch)\(\s*[\"']([^\"']+)[\"']" +) + + +def _source_routes() -> frozenset[str]: + return frozenset(chain.from_iterable(_ROUTE_DECORATOR.findall(path.read_text()) for path in SOURCE_FILES)) + + +def _route_to_regex(route: str) -> re.Pattern[str]: + def route_part(part: str) -> str: + if part.startswith("{") and part.endswith("}"): + return "[^?]+" if part.endswith(":path}") else "[^/?]+" + return re.escape(part) + + pattern: Final = "".join(route_part(part) for part in re.split(r"(\{[^}]+\})", route)) + return re.compile(f"^{pattern}$") + + +def _url_literal(node: ast.AST) -> str | None: + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + if isinstance(node, ast.JoinedStr): + return "".join( + value.value if isinstance(value, ast.Constant) and isinstance(value.value, str) else "X" + for value in node.values + ) + return None + + +def _urls(path: Path) -> Iterator[str]: + tree: Final = ast.parse(path.read_text()) + return ( + literal.split("?", 1)[0] + for node in ast.walk(tree) + if (literal := _url_literal(node)) is not None and literal.startswith("/") + ) + + +def _directory_urls(directory: Path) -> Iterator[str]: + return chain.from_iterable( + _urls(path) + for path in sorted(directory.glob("test_*.py")) + if path.resolve() != SELF + ) + + +def _test_urls() -> frozenset[str]: + return frozenset( + chain.from_iterable(_directory_urls(directory) for directory in TEST_DIRS) + ) + + +def _uncovered_routes(routes: frozenset[str], urls: frozenset[str]) -> Iterator[str]: + return ( + route + for route in routes + if not any(_route_to_regex(route).match(url) for url in urls) + ) + + +def test_every_management_route_has_a_behavior_scenario() -> None: + routes: Final = _source_routes() + assert routes, "no @router routes parsed" + + urls: Final = _test_urls() + uncovered: Final = tuple(sorted(_uncovered_routes(routes, urls))) + assert ( + not uncovered + ), "management routes with no behavior-suite scenario:\n " + "\n ".join(uncovered) diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py index 59239786584..98ea2269252 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_passthrough_endpoints.py @@ -1,8 +1,10 @@ import asyncio import importlib import json +from typing import Final import pytest +import respx import litellm from litellm import Router @@ -198,3 +200,60 @@ def setup_and_teardown(): yield loop.close() asyncio.set_event_loop(None) + + +@pytest.mark.asyncio +async def test_gemini_pass_through_endpoint_forwards_key_and_body( + respx_mock: "respx.MockRouter", httpx_transport: None, monkeypatch: pytest.MonkeyPatch +) -> None: + import httpx + from fastapi import Request, Response + + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + gemini_proxy_route, + ) + + monkeypatch.setenv("GEMINI_API_KEY", "gemini-test-key") + endpoint: Final = "v1beta/models/gemini-2.5-flash:countTokens" + upstream: Final = respx_mock.post( + url__startswith=f"https://generativelanguage.googleapis.com/{endpoint}" + ).mock( + return_value=httpx.Response( + 200, + json={"totalTokens": 9}, + ) + ) + body: Final = json.dumps( + {"contents": [{"parts": [{"text": "The quick brown fox jumps over the lazy dog."}]}]} + ).encode() + scope: Final = { + "type": "http", + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": f"/gemini/{endpoint}", + "raw_path": f"/gemini/{endpoint}".encode(), + "root_path": "", + "query_string": b"key=sk-9876", + "headers": [(b"content-type", b"application/json")], + "client": ("127.0.0.1", 51234), + "server": ("proxy.local", 4000), + "state": {}, + } + + async def receive() -> dict: + return {"type": "http.request", "body": body, "more_body": False} + + response: Final = await gemini_proxy_route( + endpoint=f"{endpoint}?key=sk-9876", + request=Request(scope, receive), + fastapi_response=Response(), + ) + + assert response.status_code == 200 + assert json.loads(response.body) == {"totalTokens": 9} + assert upstream.call_count == 1 + outbound: Final = upstream.calls[0].request + assert outbound.url.path == f"/{endpoint}" + assert outbound.url.params["key"] == "gemini-test-key" + assert json.loads(outbound.content) == json.loads(body) diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index bd66bc24837..eb31797d460 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -9911,6 +9911,93 @@ def test_init_kwargs_with_tags_in_header(mock_request, mock_user_api_key_dict): assert metadata["tags"] == ["tag1", "tag2"] +@pytest.mark.asyncio +@pytest.mark.parametrize(("operation", "stream"), (("generateContent", False), ("streamGenerateContent", True))) +async def test_vertex_passthrough_forwards_body_and_request_tags_to_spend_payload( + mock_request, operation, stream, httpx_transport +): + target: Final = ( + "https://aiplatform.googleapis.com/v1/projects/test-project/locations/global/publishers/google/" + f"models/gemini-3.1-flash-lite:{operation}" + ) + request_body: Final = {"contents": [{"role": "user", "parts": [{"text": "Say hello"}]}]} + response_body: Final = { + "candidates": [{"content": {"role": "model", "parts": [{"text": "hello"}]}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2}, + } + request: Final = mock_request( + headers=Headers({"tags": "vertex-js-sdk,pass-through-endpoint"}), + request_body=request_body, + ) + user_api_key_dict: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user", team_id="test-team") + + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post(target).respond(status_code=200, json=response_body) + response: Final = await pass_through_request( + request=request, + target=target, + custom_headers={}, + user_api_key_dict=user_api_key_dict, + custom_llm_provider="vertex_ai", + stream=stream, + ) + + assert response.status_code == 200 + if stream: + streamed_body: Final = b"".join([chunk async for chunk in response.body_iterator]) + assert json.loads(streamed_body) == response_body + assert json.loads(route.calls.last.request.content) == request_body + + start_time: Final = datetime(2026, 1, 15, 12, 0, 0) + passthrough_payload: Final = PassthroughStandardLoggingPayload( + url=target, + request_body=request_body, + request_method="POST", + ) + logging_obj: Final = LiteLLMLoggingObj( + model="gemini-3.1-flash-lite", + messages=[], + stream=stream, + call_type="pass_through_endpoint", + start_time=start_time, + litellm_call_id="vertex-request-tags", + function_id="vertex-request-tags", + ) + logging_kwargs: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( + request=request, + user_api_key_dict=user_api_key_dict, + passthrough_logging_payload=passthrough_payload, + _parsed_body=request_body, + litellm_call_id="vertex-request-tags", + logging_obj=logging_obj, + ) + logging_obj.update_environment_variables( + model="gemini-3.1-flash-lite", + user="unknown", + optional_params={}, + litellm_params=logging_kwargs["litellm_params"], + call_type="pass_through_endpoint", + ) + log_dispatch: Final = AsyncMock() + await PassThroughEndpointLogging(log_dispatch=log_dispatch).pass_through_async_success_handler( + httpx_response=route.calls.last.response, + response_body=response_body, + logging_obj=logging_obj, + url_route=target, + result=json.dumps(response_body), + start_time=start_time, + end_time=start_time, + cache_hit=False, + request_body=request_body, + custom_llm_provider="vertex_ai", + **logging_kwargs, + ) + + logged_kwargs: Final = log_dispatch.await_args.kwargs + assert logged_kwargs["call_type"] == "pass_through_endpoint" + assert logged_kwargs["litellm_params"]["metadata"]["tags"] == ["vertex-js-sdk", "pass-through-endpoint"] + + athropic_request_body: Final = { "model": "claude-sonnet-4-5-20250929", "max_tokens": 256, diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 0779f692b17..4f1cf2d8c28 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -1,18 +1,18 @@ import json +import os import re +from collections.abc import Iterator from datetime import datetime, timezone from importlib.resources import files -from collections.abc import Iterator from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest - - from fastapi import FastAPI from fastapi.testclient import TestClient from pydantic import TypeAdapter +import litellm from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.public_endpoints import router from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets @@ -1168,8 +1168,8 @@ def test_public_mcp_hub_derives_publication_metadata_without_mutating_registry( stored_public: bool | None, ) -> None: from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager - from litellm.types.mcp_server.mcp_server_manager import MCPServer from litellm.proxy._types import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer app: Final = FastAPI() app.include_router(router) @@ -1253,8 +1253,8 @@ def test_public_mcp_hub_returns_empty_when_whitelist_unset(): def test_public_mcp_hub_does_not_expose_upstream_url(): """Regression: /public/mcp_hub is unauthenticated, so the gateway-internal upstream url must never appear in its response even when the server has one.""" - from litellm.types.mcp_server.mcp_server_manager import MCPServer from litellm.proxy._types import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer app = FastAPI() app.include_router(router) @@ -1586,6 +1586,47 @@ def test_get_whats_new_local_mode_serves_bundled_list( assert len(served.launches) > 0 +@pytest.mark.asyncio +async def test_get_whats_new_serves_bundled_list_without_fetching_when_no_url_is_configured( + monkeypatch: pytest.MonkeyPatch, reset_whats_new_cache: None +) -> None: + from litellm.proxy.public_endpoints.public_endpoints import _load_bundled_whats_new, get_whats_new + + monkeypatch.delenv("LITELLM_LOCAL_WHATS_NEW", raising=False) + + async def forbidden_fetch(url: str) -> WhatsNewResponse: + raise AssertionError(f"no URL is configured, so nothing may be fetched, got {url}") + + served = await get_whats_new(url=None, fetch=forbidden_fetch) + + assert served == _load_bundled_whats_new() + assert len(served.launches) > 0 + + +def test_whats_new_url_default_mirrors_the_env_var_so_no_remote_url_is_baked_in() -> None: + assert litellm.whats_new_url == os.environ.get("LITELLM_WHATS_NEW_URL") + + +@pytest.mark.asyncio +async def test_public_whats_new_route_serves_the_bundled_list_without_fetching_when_no_url_is_configured( + monkeypatch: pytest.MonkeyPatch, reset_whats_new_cache: None +) -> None: + from litellm.proxy.public_endpoints import public_endpoints + from litellm.proxy.public_endpoints.public_endpoints import _load_bundled_whats_new, get_public_whats_new + + monkeypatch.delenv("LITELLM_LOCAL_WHATS_NEW", raising=False) + + async def forbidden_fetch(url: str) -> bytes: + raise AssertionError(f"no URL is configured, so the route may not fetch, got {url}") + + monkeypatch.setattr(public_endpoints, "_fetch_remote_bytes", forbidden_fetch) + + served = await get_public_whats_new() + + assert served == _load_bundled_whats_new() + assert len(served.launches) > 0 + + @pytest.mark.asyncio async def test_get_whats_new_fetches_once_per_process( monkeypatch: pytest.MonkeyPatch, reset_whats_new_cache: None diff --git a/tests/proxy_unit_tests/test_configs/custom_callbacks.py b/tests/unit/proxy/test_configs/custom_callbacks.py similarity index 98% rename from tests/proxy_unit_tests/test_configs/custom_callbacks.py rename to tests/unit/proxy/test_configs/custom_callbacks.py index c7d66c068e9..20d8088b30d 100644 --- a/tests/proxy_unit_tests/test_configs/custom_callbacks.py +++ b/tests/unit/proxy/test_configs/custom_callbacks.py @@ -1,3 +1,5 @@ +import threading + from litellm.integrations.custom_logger import CustomLogger import inspect import litellm @@ -20,6 +22,7 @@ class testCustomCallbackProxy(CustomLogger): self.async_embedding_kwargs_fail = None # type: ignore self.streaming_response_obj = None # type: ignore + self.done_event = threading.Event() blue_color_code = "\033[94m" reset_color_code = "\033[0m" print(f"{blue_color_code}Initialized LiteLLM custom logger") @@ -74,6 +77,7 @@ class testCustomCallbackProxy(CustomLogger): self.streaming_response_obj = response_obj self.async_completion_kwargs = kwargs + self.done_event.set() model = kwargs.get("model", None) messages = kwargs.get("messages", None) diff --git a/tests/proxy_unit_tests/test_configs/test_bad_config.yaml b/tests/unit/proxy/test_configs/test_bad_config.yaml similarity index 70% rename from tests/proxy_unit_tests/test_configs/test_bad_config.yaml rename to tests/unit/proxy/test_configs/test_bad_config.yaml index 4bc4c7cc541..72ceefcda2d 100644 --- a/tests/proxy_unit_tests/test_configs/test_bad_config.yaml +++ b/tests/unit/proxy/test_configs/test_bad_config.yaml @@ -6,16 +6,15 @@ model_list: - model_name: working-azure-gpt-3.5-turbo litellm_params: model: azure/gpt-4.1-mini - api_base: os.environ/AZURE_AI_API_BASE - api_key: os.environ/AZURE_AI_API_KEY + api_base: https://azure-unit.example.com + api_key: bad-key - model_name: azure-gpt-3.5-turbo litellm_params: model: azure/gpt-4.1-mini - api_base: os.environ/AZURE_AI_API_BASE + api_base: https://azure-unit.example.com api_key: bad-key - model_name: azure-embedding litellm_params: model: azure/text-embedding-ada-002 - api_base: os.environ/AZURE_AI_API_BASE + api_base: https://azure-unit.example.com api_key: bad-key - \ No newline at end of file diff --git a/tests/proxy_unit_tests/test_configs/test_custom_logger.yaml b/tests/unit/proxy/test_configs/test_custom_logger.yaml similarity index 70% rename from tests/proxy_unit_tests/test_configs/test_custom_logger.yaml rename to tests/unit/proxy/test_configs/test_custom_logger.yaml index 6ff1b086357..c58df01b602 100644 --- a/tests/proxy_unit_tests/test_configs/test_custom_logger.yaml +++ b/tests/unit/proxy/test_configs/test_custom_logger.yaml @@ -1,8 +1,8 @@ -model_list: +model_list: - model_name: Azure OpenAI GPT-4 Canada litellm_params: model: gpt-4.1-nano - api_key: os.environ/OPENAI_API_KEY + api_key: sk-unit-fake model_info: mode: chat input_cost_per_token: 0.0002 @@ -10,7 +10,7 @@ model_list: - model_name: azure-embedding-model litellm_params: model: text-embedding-ada-002 - api_key: os.environ/OPENAI_API_KEY + api_key: sk-unit-fake model_info: mode: embedding input_cost_per_token: 0.002 @@ -18,5 +18,4 @@ model_list: litellm_settings: drop_params: True - set_verbose: True - callbacks: custom_callbacks.my_custom_logger \ No newline at end of file + callbacks: custom_callbacks.my_custom_logger diff --git a/tests/unit/proxy/test_proxy_server_custom_logger.py b/tests/unit/proxy/test_proxy_server_custom_logger.py new file mode 100644 index 00000000000..a9cf111880e --- /dev/null +++ b/tests/unit/proxy/test_proxy_server_custom_logger.py @@ -0,0 +1,203 @@ +import asyncio +import json +import os +from pathlib import Path +from typing import Final + +import httpx +import pytest +import respx +from fastapi.testclient import TestClient + +import litellm + +CONFIG_DIR: Final = Path(__file__).resolve().parent / "test_configs" +CUSTOM_CALLBACKS_PATH: Final = CONFIG_DIR / "custom_callbacks.py" +OPENAI_BASE: Final = "https://api.openai.com/v1" + +CHAT_COMPLETION_BODY: Final = { + "id": "chatcmpl-unit-1", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-4.1-nano", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "litellm poem"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 3, "total_tokens": 12}, +} + +EMBEDDING_BODY: Final = { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], + "model": "text-embedding-ada-002", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, +} + +STREAM_CHUNK_1: Final = { + "id": "chatcmpl-unit-stream", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "gpt-4.1-nano", + "choices": [ + {"index": 0, "delta": {"role": "assistant", "content": "Lite"}, "finish_reason": None} + ], +} +STREAM_CHUNK_2: Final = { + "id": "chatcmpl-unit-stream", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "gpt-4.1-nano", + "choices": [{"index": 0, "delta": {"content": "LLM"}, "finish_reason": "stop"}], +} +STREAM_BODY: Final = ( + f"data: {json.dumps(STREAM_CHUNK_1)}\n\n" + f"data: {json.dumps(STREAM_CHUNK_2)}\n\n" + "data: [DONE]\n\n" +).encode() + + +@pytest.fixture +def client(monkeypatch: pytest.MonkeyPatch, httpx_transport: None) -> TestClient: + from litellm.proxy.proxy_server import ( + app, + cleanup_router_config_variables, + initialize, + ) + + monkeypatch.setenv("LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY", "true") + cleanup_router_config_variables() + asyncio.run(initialize(config=str(CONFIG_DIR / "test_custom_logger.yaml"))) + return TestClient(app) + + +def _custom_logger() -> object: + from litellm.proxy.types_utils.utils import get_instance_fn + + return get_instance_fn( + value="custom_callbacks.my_custom_logger", + config_file_path=str(CUSTOM_CALLBACKS_PATH), + ) + + +def test_chat_completion(client: TestClient, respx_mock: respx.MockRouter) -> None: + upstream: Final = respx_mock.post(f"{OPENAI_BASE}/chat/completions").mock( + return_value=httpx.Response(200, json=CHAT_COMPLETION_BODY) + ) + my_custom_logger = _custom_logger() + litellm.callbacks = [my_custom_logger] + + response: Final = client.post( + "/chat/completions", + json={ + "model": "Azure OpenAI GPT-4 Canada", + "messages": [{"role": "user", "content": "write a litellm poem"}], + "max_tokens": 10, + }, + ) + + assert response.status_code == 200, response.text + assert my_custom_logger.done_event.wait(10) + assert my_custom_logger.async_success is True + assert my_custom_logger.async_completion_kwargs["model"] == "gpt-4.1-nano" + litellm_params: Final = my_custom_logger.async_completion_kwargs["litellm_params"] + + metadata: Final = litellm_params.get("metadata") + assert "user_api_key" in metadata + assert "user_api_key_metadata" in metadata + assert "headers" in metadata + + model_info: Final = litellm_params.get("model_info") + assert model_info.get("id") == "gm" + assert model_info.get("mode") == "chat" + assert model_info.get("input_cost_per_token") == 0.0002 + + proxy_server_request: Final = litellm_params.get("proxy_server_request") + assert proxy_server_request.get("url") == "http://testserver/chat/completions" + assert proxy_server_request.get("method") == "POST" + assert "authorization" not in proxy_server_request["headers"] + body: Final = proxy_server_request["body"] + assert body.get("model") == "Azure OpenAI GPT-4 Canada" + assert body.get("messages") == [ + {"role": "user", "content": "write a litellm poem"} + ] + assert body.get("max_tokens") == 10 + assert upstream.call_count == 1 + + +def test_chat_completion_stream(client: TestClient, respx_mock: respx.MockRouter) -> None: + upstream: Final = respx_mock.post(f"{OPENAI_BASE}/chat/completions").mock( + return_value=httpx.Response( + 200, content=STREAM_BODY, headers={"content-type": "text/event-stream"} + ) + ) + my_custom_logger = _custom_logger() + litellm.callbacks = [my_custom_logger] + assert my_custom_logger.streaming_response_obj is None + + response: Final = client.post( + "/chat/completions", + json={ + "model": "Azure OpenAI GPT-4 Canada", + "messages": [{"role": "user", "content": "write 1 line poem about LiteLLM"}], + "max_tokens": 40, + "stream": True, + }, + ) + + assert response.status_code == 200, response.text + complete_response: Final = "".join( + json.loads(data)["choices"][0]["delta"].get("content") or "" + for data in ( + line.replace("data: ", "") for line in response.iter_lines() if line + ) + if "[DONE]" not in data + ) + + assert complete_response == "LiteLLM" + assert my_custom_logger.done_event.wait(10) + assert ( + complete_response + == my_custom_logger.streaming_response_obj["choices"][0]["message"]["content"] + ) + assert upstream.call_count == 1 + + +def test_embedding(client: TestClient, respx_mock: respx.MockRouter) -> None: + upstream: Final = respx_mock.post(f"{OPENAI_BASE}/embeddings").mock( + return_value=httpx.Response(200, json=EMBEDDING_BODY) + ) + my_custom_logger = _custom_logger() + litellm.callbacks = [my_custom_logger] + assert my_custom_logger.async_success_embedding is False + + response: Final = client.post( + "/embeddings", + json={"model": "azure-embedding-model", "input": ["hello"]}, + ) + + assert response.status_code == 200, response.text + assert my_custom_logger.done_event.wait(10) + assert my_custom_logger.async_success_embedding is True + assert my_custom_logger.async_embedding_kwargs["model"] == "text-embedding-ada-002" + litellm_params: Final = my_custom_logger.async_embedding_kwargs["litellm_params"] + + metadata: Final = litellm_params.get("metadata") + assert "user_api_key" in metadata + assert "headers" in metadata + + proxy_server_request: Final = litellm_params.get("proxy_server_request") + assert proxy_server_request.get("url") == "http://testserver/embeddings" + assert proxy_server_request.get("method") == "POST" + body: Final = proxy_server_request["body"] + assert body.get("model") == "azure-embedding-model" + assert body.get("input") == ["hello"] + + model_info: Final = litellm_params.get("model_info") + assert model_info.get("mode") == "embedding" + assert model_info.get("id") == "hello" + assert model_info.get("input_cost_per_token") == 0.002 + assert upstream.call_count == 1 diff --git a/tests/unit/proxy/test_proxy_server_exception_mapping.py b/tests/unit/proxy/test_proxy_server_exception_mapping.py new file mode 100644 index 00000000000..dbb595f129a --- /dev/null +++ b/tests/unit/proxy/test_proxy_server_exception_mapping.py @@ -0,0 +1,211 @@ +import asyncio +import json +from pathlib import Path +from typing import Final + +import httpx +import openai +import pytest +import respx +from fastapi.testclient import TestClient + +CONFIG_DIR: Final = Path(__file__).resolve().parent / "test_configs" +OPENAI_CHAT_COMPLETIONS: Final = "https://api.openai.com/v1/chat/completions" +AZURE_CHAT_COMPLETIONS: Final = ( + "https://azure-unit.example.com/openai/deployments/gpt-4.1-mini/chat/completions" +) +AZURE_EMBEDDINGS: Final = ( + "https://azure-unit.example.com/openai/deployments/text-embedding-ada-002/embeddings" +) + +AZURE_401_BODY: Final = { + "error": {"code": "401", "message": "Access denied due to invalid subscription key."} +} +CONTEXT_LENGTH_EXCEEDED_BODY: Final = { + "error": { + "message": "This model's maximum context length is 4096 tokens. However, your messages resulted in 10007 tokens. Please reduce the length of the messages.", + "type": "invalid_request_error", + "param": "messages", + "code": "context_length_exceeded", + } +} + + +@pytest.fixture +def client(monkeypatch: pytest.MonkeyPatch, httpx_transport: None) -> TestClient: + from litellm.proxy.proxy_server import ( + app, + cleanup_router_config_variables, + initialize, + ) + + monkeypatch.setenv("LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY", "true") + cleanup_router_config_variables() + asyncio.run(initialize(config=str(CONFIG_DIR / "test_bad_config.yaml"))) + return TestClient(app) + + +def _openai_exception(response: httpx.Response) -> openai.APIStatusError: + return openai.OpenAI(api_key="anything")._make_status_error_from_response( + response=response + ) + + +def test_chat_completion_exception( + client: TestClient, respx_mock: respx.MockRouter +) -> None: + respx_mock.post(OPENAI_CHAT_COMPLETIONS).mock( + return_value=httpx.Response( + 401, + json={ + "error": { + "message": "Incorrect API key provided: bad-key. You can find your API key at https://platform.openai.com/account/api-keys.", + "type": "invalid_request_error", + "param": None, + "code": "invalid_api_key", + } + }, + ) + ) + + response: Final = client.post( + "/chat/completions", + json={ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 10, + }, + ) + + json_response: Final = response.json() + assert json_response.keys() == {"error"} + assert "litellm.AuthenticationError: AuthenticationError" in json_response["error"]["message"] + assert type(json_response["error"]["code"]) is str + assert isinstance(_openai_exception(response), openai.AuthenticationError) + + +def test_chat_completion_exception_azure( + client: TestClient, respx_mock: respx.MockRouter +) -> None: + upstream: Final = respx_mock.post(AZURE_CHAT_COMPLETIONS).mock( + return_value=httpx.Response(401, json=AZURE_401_BODY) + ) + + response: Final = client.post( + "/chat/completions", + json={ + "model": "azure-gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 10, + }, + ) + + assert response.status_code == 401, response.text + json_response: Final = response.json() + assert json_response.keys() == {"error"} + assert "litellm.AuthenticationError" in json_response["error"]["message"] + assert upstream.call_count == 1 + outbound: Final = upstream.calls[0].request + assert json.loads(outbound.content)["messages"] == [ + {"role": "user", "content": "hi"} + ] + assert outbound.headers["api-key"] == "bad-key" + assert isinstance(_openai_exception(response), openai.AuthenticationError) + + +def test_embedding_auth_exception_azure( + client: TestClient, respx_mock: respx.MockRouter +) -> None: + upstream: Final = respx_mock.post(AZURE_EMBEDDINGS).mock( + return_value=httpx.Response(401, json=AZURE_401_BODY) + ) + + response: Final = client.post( + "/embeddings", + json={"model": "azure-embedding", "input": ["hi"]}, + ) + + assert response.status_code == 401, response.text + json_response: Final = response.json() + assert json_response.keys() == {"error"} + assert upstream.call_count == 1 + outbound: Final = upstream.calls[0].request + assert json.loads(outbound.content)["input"] == ["hi"] + assert outbound.headers["api-key"] == "bad-key" + assert isinstance(_openai_exception(response), openai.AuthenticationError) + + +def test_exception_openai_bad_model(client: TestClient) -> None: + response: Final = client.post( + "/chat/completions", + json={ + "model": "azure/GPT-12", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 10, + }, + ) + + json_response: Final = response.json() + assert json_response.keys() == {"error"} + assert isinstance(_openai_exception(response), openai.BadRequestError) + + +def test_chat_completion_exception_any_model(client: TestClient) -> None: + response: Final = client.post( + "/chat/completions", + json={ + "model": "Lite-GPT-12", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 10, + }, + ) + + json_response: Final = response.json() + assert json_response.keys() == {"error"} + openai_exception: Final = _openai_exception(response) + assert isinstance(openai_exception, openai.BadRequestError) + assert "/chat/completions: Invalid model name passed in model=Lite-GPT-12" in str( + openai_exception.message + ) + + +def test_embedding_exception_any_model(client: TestClient) -> None: + response: Final = client.post( + "/embeddings", + json={"model": "Lite-GPT-12", "input": ["hi"]}, + ) + + json_response: Final = response.json() + assert json_response.keys() == {"error"} + openai_exception: Final = _openai_exception(response) + assert isinstance(openai_exception, openai.BadRequestError) + assert "/embeddings: Invalid model name passed in model=Lite-GPT-12" in str( + openai_exception.message + ) + + +def test_chat_completion_exception_azure_context_window( + client: TestClient, respx_mock: respx.MockRouter +) -> None: + upstream: Final = respx_mock.post(AZURE_CHAT_COMPLETIONS).mock( + return_value=httpx.Response(400, json=CONTEXT_LENGTH_EXCEEDED_BODY) + ) + + response: Final = client.post( + "/chat/completions", + json={ + "model": "working-azure-gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hi" * 10000}], + "max_tokens": 10, + }, + ) + + assert response.status_code == 400, response.text + json_response: Final = response.json() + assert json_response.keys() == {"error"} + assert upstream.call_count == 1 + error: Final = json_response["error"] + assert "ContextWindowExceededError" in error["message"] + assert "maximum context length is 4096 tokens" in error["message"] + assert type(error["code"]) is str + assert isinstance(_openai_exception(response), openai.BadRequestError) diff --git a/tests/unit/router_strategy/test_lowest_tpm_rpm_v2.py b/tests/unit/router_strategy/test_lowest_tpm_rpm_v2.py index bbba7e5e20e..129bb10eccc 100644 --- a/tests/unit/router_strategy/test_lowest_tpm_rpm_v2.py +++ b/tests/unit/router_strategy/test_lowest_tpm_rpm_v2.py @@ -43,7 +43,7 @@ def restore_verbose_logger_level() -> Iterator[None]: @pytest.mark.usefixtures("restore_verbose_logger_level") def test_tpm_rpm_updated(): - from tests.local_testing.create_mock_standard_logging_payload import create_standard_logging_payload + from tests.unit.create_mock_standard_logging_payload import create_standard_logging_payload test_cache = DualCache() lowest_tpm_logger = LowestTPMLoggingHandler(router_cache=test_cache) @@ -98,7 +98,7 @@ def test_tpm_rpm_updated(): @pytest.mark.usefixtures("restore_verbose_logger_level") def test_get_available_deployments(): - from tests.local_testing.create_mock_standard_logging_payload import create_standard_logging_payload + from tests.unit.create_mock_standard_logging_payload import create_standard_logging_payload test_cache = DualCache() model_list = [ @@ -190,7 +190,7 @@ def test_router_get_available_deployments(): """ Test if routers 'get_available_deployments' returns the lowest tpm deployment """ - from tests.local_testing.create_mock_standard_logging_payload import create_standard_logging_payload + from tests.unit.create_mock_standard_logging_payload import create_standard_logging_payload model_list = [ { @@ -287,7 +287,7 @@ def test_router_skip_rate_limited_deployments(): """ Test if routers 'get_available_deployments' raises No Models Available error if max tpm would be reached by message """ - from tests.local_testing.create_mock_standard_logging_payload import create_standard_logging_payload + from tests.unit.create_mock_standard_logging_payload import create_standard_logging_payload model_list = [ { diff --git a/tests/unit/router_utils/test_prompt_caching_cache.py b/tests/unit/router_utils/test_prompt_caching_cache.py index b0bf03498d4..3a1132354c4 100644 --- a/tests/unit/router_utils/test_prompt_caching_cache.py +++ b/tests/unit/router_utils/test_prompt_caching_cache.py @@ -1,8 +1,19 @@ +import asyncio from typing import Final +import pytest + from litellm.router_utils.prompt_caching_cache import PromptCachingCache +async def _first_non_none(read): + while True: + value = await read() + if value is not None: + return value + await asyncio.sleep(0) + + def test_extract_cacheable_prefix_with_string_content_and_message_level_cache_control(): """ Test that extract_cacheable_prefix correctly handles messages where: @@ -87,3 +98,89 @@ def test_extract_cacheable_prefix_mixed_string_and_list_content(): assert result[0]["role"] == "system" assert result[1]["content"] == "First cached message" assert isinstance(result[2]["content"], list) + + +@pytest.mark.asyncio +async def test_same_cacheable_prefix_routes_to_same_deployment() -> None: + from litellm import Router + + def _messages(user_content: str): + return [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are an AI assistant tasked with analyzing legal documents.", + }, + { + "type": "text", + "text": "Here is the full text of a complex legal agreement" * 400, + "cache_control": {"type": "ephemeral"}, + }, + ], + }, + {"role": "user", "content": user_content}, + ] + + router: Final = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-5-mini", + "api_key": f"test-key-{i}", + }, + "model_info": {"id": f"deployment-{i}"}, + } + for i in range(1, 4) + ], + routing_strategy="simple-shuffle", + optional_pre_call_checks=["prompt_caching"], + ) + + messages1: Final = _messages("what are the key terms and conditions in this agreement?") + messages2: Final = _messages("how many words are there?") + messages3: Final = _messages("how many sentences are there?") + + key1: Final = PromptCachingCache.get_prompt_caching_cache_key(messages1, None) + key2: Final = PromptCachingCache.get_prompt_caching_cache_key(messages2, None) + key3: Final = PromptCachingCache.get_prompt_caching_cache_key(messages3, None) + + assert key1 is not None + assert key1 == key2 == key3 + + response1: Final = await router.acompletion( + model="test-model", messages=messages1, mock_response="ok" + ) + model_id_1: Final = response1._hidden_params["model_id"] + assert model_id_1 in {"deployment-1", "deployment-2", "deployment-3"} + + from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + ) + + affinity_messages: Final = AnthropicCacheControlHook.messages_with_default_injections( + messages=messages2, + models=("gpt-5-mini",), + tools=None, + enable_prompt_caching=None, + request_kwargs={}, + ) + cache: Final = PromptCachingCache(cache=router.cache) + cached_2: Final = await asyncio.wait_for( + _first_non_none(lambda: cache.async_get_model_id(affinity_messages, None)), + timeout=10, + ) + assert cached_2 == {"model_id": model_id_1} + + response2: Final = await router.acompletion( + model="test-model", messages=messages2, mock_response="ok" + ) + response3: Final = await router.acompletion( + model="test-model", messages=messages3, mock_response="ok" + ) + model_id_2: Final = response2._hidden_params["model_id"] + model_id_3: Final = response3._hidden_params["model_id"] + + assert model_id_1 == model_id_2 == model_id_3 diff --git a/tests/unit/test_assert_ci_coverage.py b/tests/unit/test_assert_ci_coverage.py index 4c9fe1739e6..443adf86d9a 100644 --- a/tests/unit/test_assert_ci_coverage.py +++ b/tests/unit/test_assert_ci_coverage.py @@ -124,7 +124,6 @@ def test_every_circleci_postgres_suite_is_credited_on_the_repo_as_it_stands() -> if isinstance(job, dict) and "postgres_suite" in job ) assert paths == ( - "tests/proxy_behavior", "tests/proxy_security_tests", "tests/proxy_migration_tests", "tests/integration/database/test_roi_observed.py", @@ -433,7 +432,7 @@ def test_a_dockerfile_directory_entry_is_stale_because_only_an_exact_path_exempt def test_a_workflow_that_names_a_file_clears_it_from_the_slice_check(): named = coverage._workflow_named_tokens() assert named, "the workflows must name some test paths or the check proves nothing" - assert any(coverage._token_covers(token, "tests/proxy_unit_tests/test_proxy_custom_logger.py") for token in named) + assert any(coverage._token_covers(token, "tests/unit/proxy/test_proxy_server_custom_logger.py") for token in named) def test_the_slice_check_credits_only_workflows_never_the_circleci_config(): diff --git a/tests/test_litellm/test_check_unbounded_in_lists.py b/tests/unit/test_check_unbounded_in_lists.py similarity index 100% rename from tests/test_litellm/test_check_unbounded_in_lists.py rename to tests/unit/test_check_unbounded_in_lists.py diff --git a/tests/unit/test_circleci_path_filter.py b/tests/unit/test_circleci_path_filter.py index fa30d878f9f..b432f358f85 100644 --- a/tests/unit/test_circleci_path_filter.py +++ b/tests/unit/test_circleci_path_filter.py @@ -59,22 +59,6 @@ CI = [".github/workflows/test-unit.yml"] ("mcp-dependencies", [".circleci/scripts/classify_changes.sh"], "run"), ("mcp-dependencies", ["litellm/llms/openai/chat/gpt_transformation.py"], "skip"), ("mcp-dependencies", DOCS + CLIENT, "skip"), - ("provider-harness", ["tests/e2e/provider_cache.py"], "run"), - ("provider-harness", ["tests/e2e/conftest.py"], "run"), - ("provider-harness", ["tests/e2e/e2e_http.py"], "run"), - ("provider-harness", ["tests/e2e_harness/test_provider_edge.py"], "run"), - ("provider-harness", ["tests/e2e_harness/logging/test_datadog_reader.py"], "skip"), - ("provider-harness", ["tests/code_coverage_tests/test_provider_cache.py"], "run"), - ("provider-harness", ["tests/code_coverage_tests/test_provider_replay_harness.py"], "run"), - ("provider-harness", [".circleci/config.yml"], "run"), - ("provider-harness", [".circleci/scripts/classify_changes.sh"], "run"), - ("provider-harness", ["pyproject.toml"], "run"), - ("provider-harness", ["uv.lock"], "run"), - ("provider-harness", ["tests/e2e/PROVIDER_CACHE.md"], "skip"), - ("provider-harness", ["tests/e2e/ui/test_example.py"], "skip"), - ("provider-harness", ["tests/e2e/quota_management/test_quota.py"], "skip"), - ("provider-harness", ["litellm/main.py"], "skip"), - ("provider-harness", ["ui/litellm-dashboard/src/App.tsx"], "skip"), ("windows-release", ["litellm-rust/crates/inference/src/lib.rs"], "run"), ("windows-release", ["litellm/rust_bridge/dispatch.py"], "run"), ("windows-release", ["rust-toolchain.toml"], "run"), @@ -113,11 +97,6 @@ CI = [".github/workflows/test-unit.yml"] ("cost-map-only", ["model_prices_and_context_window.json"], "run"), ("cost-map-only", ["litellm/model_prices_and_context_window_backup.json"], "run"), ("cost-map-only", ["model_prices_and_context_window.schema.json"], "run"), - ( - "cost-map-only", - ["model_prices_and_context_window.json", "tests/test_litellm/test_x.py"], - "run", - ), ( "cost-map-only", ["model_prices_and_context_window.json", "tests/unit/proxy/test_y.py"], @@ -128,7 +107,7 @@ CI = [".github/workflows/test-unit.yml"] ["model_prices_and_context_window.json", "litellm/utils.py"], "skip", ), - ("cost-map-only", ["tests/test_litellm/test_x.py"], "skip"), + ("cost-map-only", ["tests/unit/proxy/test_y.py"], "skip"), ("cost-map-only", ["model_prices_and_context_window.json", "docs/pricing.md"], "skip"), ("cost-map-only", ["model_prices_and_context_window.json", "docs/foo.mdx"], "skip"), ("cost-map-only", [], "skip"), diff --git a/tests/test_litellm/tracing/test_otlp_http.py b/tests/unit/tracing/test_otlp_http.py similarity index 100% rename from tests/test_litellm/tracing/test_otlp_http.py rename to tests/unit/tracing/test_otlp_http.py diff --git a/tests/vector_store_tests/base_vector_store_test.py b/tests/vector_store_tests/base_vector_store_test.py deleted file mode 100644 index 1e61613decf..00000000000 --- a/tests/vector_store_tests/base_vector_store_test.py +++ /dev/null @@ -1,318 +0,0 @@ -import httpx -import json -import pytest -from typing import Any, Dict, List -from unittest.mock import MagicMock, Mock, patch -from litellm._uuid import uuid -import time -import base64 - -import litellm -from abc import ABC, abstractmethod -from litellm.integrations.custom_logger import CustomLogger -from litellm.types.utils import StandardLoggingPayload - - -class BaseVectorStoreTest(ABC): - """ - Abstract base test class that enforces a common test across all test classes. - """ - - @abstractmethod - def get_base_request_args(self) -> dict: - """Must return the base request args""" - pass - - @abstractmethod - def get_base_create_vector_store_args(self) -> dict: - """Must return the base create vector store args""" - pass - - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - async def test_basic_search_vector_store(self, sync_mode): - litellm.turn_on_debug() - litellm.set_verbose = True - base_request_args = self.get_base_request_args() - default_query = base_request_args.pop("query", "Basic ping") - try: - if sync_mode: - response = litellm.vector_stores.search( - query=default_query, **base_request_args - ) - else: - response = await litellm.vector_stores.asearch( - query=default_query, **base_request_args - ) - except litellm.InternalServerError: - pytest.skip("Skipping test due to litellm.InternalServerError") - - print("litellm response=", json.dumps(response, indent=4, default=str)) - - # Validate response structure - self._validate_vector_store_response(response) - - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - async def test_basic_create_vector_store(self, sync_mode): - litellm.turn_on_debug() - litellm.set_verbose = True - base_request_args = self.get_base_create_vector_store_args() - - # Extract custom_llm_provider from base args if present - create_args = base_request_args - try: - if sync_mode: - response = litellm.vector_stores.create( - name="Test Vector Store", **create_args - ) - else: - response = await litellm.vector_stores.acreate( - name="Test Vector Store", **create_args - ) - except litellm.InternalServerError: - pytest.skip("Skipping test due to litellm.InternalServerError") - except Exception as e: - # If this is an authentication or permission error, skip the test - if ( - "authentication" in str(e).lower() - or "permission" in str(e).lower() - or "unauthorized" in str(e).lower() - ): - pytest.skip( - f"Skipping test due to authentication/permission error: {e}" - ) - raise - - print("litellm create response=", json.dumps(response, indent=4, default=str)) - - # Validate response structure - self._validate_vector_store_create_response(response) - - def _validate_vector_store_response(self, response): - """Validate the structure and content of a vector store search response""" - - # Check that response is a dictionary - assert isinstance( - response, dict - ), f"Response should be a dict, got {type(response)}" - - # Check required top-level fields - required_fields = ["object", "search_query", "data"] - for field in required_fields: - assert field in response, f"Missing required field '{field}' in response" - - # Validate object field - assert ( - response["object"] == "vector_store.search_results.page" - ), f"Expected object to be 'vector_store.search_results.page', got '{response['object']}'" - - # Validate search_query field - assert isinstance( - response["search_query"], str - ), f"search_query should be a list, got {type(response['search_query'])}" - assert len(response["search_query"]) > 0, "search_query should not be empty" - assert all( - isinstance(query, str) for query in response["search_query"] - ), "All items in search_query should be strings" - - # Validate data field - assert isinstance( - response["data"], list - ), f"data should be a list, got {type(response['data'])}" - - # Validate each result in data - for i, result in enumerate(response["data"]): - self._validate_search_result(result, i) - - print( - f"✅ Response validation passed: Found {len(response['data'])} search results" - ) - - def _validate_vector_store_create_response(self, response): - """Validate the structure and content of a vector store create response""" - - # Check that response is a dictionary - assert isinstance( - response, dict - ), f"Response should be a dict, got {type(response)}" - - # Check required top-level fields for create response - required_fields = ["id", "object", "created_at"] - for field in required_fields: - assert ( - field in response - ), f"Missing required field '{field}' in create response" - - # Validate object field - assert ( - response["object"] == "vector_store" - ), f"Expected object to be 'vector_store', got '{response['object']}'" - - # Validate id field - assert isinstance( - response["id"], str - ), f"id should be a string, got {type(response['id'])}" - assert len(response["id"]) > 0, "id should not be empty" - assert response["id"].startswith( - "vs_" - ), f"id should start with 'vs_', got '{response['id']}'" - - # Validate created_at field - assert isinstance( - response["created_at"], int - ), f"created_at should be an integer, got {type(response['created_at'])}" - assert response["created_at"] > 0, "created_at should be a positive timestamp" - - # Validate optional fields if present - if "name" in response: - assert isinstance( - response["name"], str - ), f"name should be a string, got {type(response['name'])}" - - if "bytes" in response: - assert isinstance( - response["bytes"], int - ), f"bytes should be an integer, got {type(response['bytes'])}" - assert response["bytes"] >= 0, "bytes should be non-negative" - - if "file_counts" in response: - self._validate_file_counts(response["file_counts"]) - - if "status" in response: - valid_statuses = ["expired", "in_progress", "completed"] - assert ( - response["status"] in valid_statuses - ), f"status should be one of {valid_statuses}, got '{response['status']}'" - - if "expires_at" in response and response["expires_at"] is not None: - assert isinstance( - response["expires_at"], int - ), f"expires_at should be an integer, got {type(response['expires_at'])}" - - if "last_active_at" in response and response["last_active_at"] is not None: - assert isinstance( - response["last_active_at"], int - ), f"last_active_at should be an integer, got {type(response['last_active_at'])}" - - if "metadata" in response and response["metadata"] is not None: - assert isinstance( - response["metadata"], dict - ), f"metadata should be a dict, got {type(response['metadata'])}" - - print( - f"✅ Create response validation passed: Vector store '{response['id']}' created successfully" - ) - - def _validate_file_counts(self, file_counts): - """Validate file_counts structure""" - assert isinstance( - file_counts, dict - ), f"file_counts should be a dict, got {type(file_counts)}" - - required_count_fields = [ - "in_progress", - "completed", - "failed", - "cancelled", - "total", - ] - for field in required_count_fields: - assert ( - field in file_counts - ), f"Missing required field '{field}' in file_counts" - assert isinstance( - file_counts[field], int - ), f"{field} should be an integer, got {type(file_counts[field])}" - assert file_counts[field] >= 0, f"{field} should be non-negative" - - # Validate that total equals sum of other counts - calculated_total = ( - file_counts["in_progress"] - + file_counts["completed"] - + file_counts["failed"] - + file_counts["cancelled"] - ) - assert ( - file_counts["total"] == calculated_total - ), f"total should equal sum of other counts ({calculated_total}), got {file_counts['total']}" - - def _validate_search_result(self, result, index): - """Validate an individual search result""" - - # Check that result is a dictionary - assert isinstance( - result, dict - ), f"Result {index} should be a dict, got {type(result)}" - - # Check required fields in each result - required_result_fields = [ - "file_id", - "filename", - "score", - "attributes", - "content", - ] - for field in required_result_fields: - assert ( - field in result - ), f"Missing required field '{field}' in result {index}" - - # Validate file_id - assert isinstance( - result["file_id"], str - ), f"file_id should be a string, got {type(result['file_id'])} in result {index}" - assert ( - len(result["file_id"]) > 0 - ), f"file_id should not be empty in result {index}" - - # Validate filename - assert isinstance( - result["filename"], str - ), f"filename should be a string, got {type(result['filename'])} in result {index}" - assert ( - len(result["filename"]) > 0 - ), f"filename should not be empty in result {index}" - - # Validate score - assert isinstance( - result["score"], (int, float) - ), f"score should be a number, got {type(result['score'])} in result {index}" - assert ( - 0.0 <= result["score"] <= 1.0 - ), f"score should be between 0.0 and 1.0, got {result['score']} in result {index}" - - # Validate attributes - assert isinstance( - result["attributes"], dict - ), f"attributes should be a dict, got {type(result['attributes'])} in result {index}" - - # Validate content - assert isinstance( - result["content"], list - ), f"content should be a list, got {type(result['content'])} in result {index}" - assert ( - len(result["content"]) > 0 - ), f"content should not be empty in result {index}" - - # Validate each content item - for j, content_item in enumerate(result["content"]): - assert isinstance( - content_item, dict - ), f"Content item {j} in result {index} should be a dict, got {type(content_item)}" - assert ( - "type" in content_item - ), f"Content item {j} in result {index} missing 'type' field" - assert ( - "text" in content_item - ), f"Content item {j} in result {index} missing 'text' field" - assert isinstance( - content_item["text"], str - ), f"Content text should be a string in item {j} of result {index}" - assert ( - len(content_item["text"]) > 0 - ), f"Content text should not be empty in item {j} of result {index}" - - print( - f"✅ Result {index} validation passed: {result['filename']} (score: {result['score']:.4f})" - ) diff --git a/tests/vector_store_tests/conftest.py b/tests/vector_store_tests/conftest.py deleted file mode 100644 index 8c1e70b14bc..00000000000 --- a/tests/vector_store_tests/conftest.py +++ /dev/null @@ -1,53 +0,0 @@ -# conftest.py - -import importlib -import os - -import pytest - -import litellm - - -@pytest.fixture(scope="function", autouse=True) -def setup_and_teardown(): - """ - This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. - """ - curr_dir = os.getcwd() # Get the current working directory - - from litellm import Router - - importlib.reload(litellm) - - try: - if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - importlib.reload(litellm.proxy.proxy_server) - except Exception as e: - print(f"Error reloading litellm.proxy.proxy_server: {e}") - - import asyncio - - loop = asyncio.get_event_loop_policy().new_event_loop() - asyncio.set_event_loop(loop) - print(litellm) - # from litellm import Router, completion, aembedding, acompletion, embedding - yield - - # Teardown code (executes after the yield point) - loop.close() # Close the loop created earlier - asyncio.set_event_loop(None) # Remove the reference to the loop - - -def pytest_collection_modifyitems(config, items): - # Separate tests in 'test_amazing_proxy_custom_logger.py' and other tests - custom_logger_tests = [ - item for item in items if "custom_logger" in item.parent.name - ] - other_tests = [item for item in items if "custom_logger" not in item.parent.name] - - # Sort tests based on their names - custom_logger_tests.sort(key=lambda x: x.name) - other_tests.sort(key=lambda x: x.name) - - # Reorder the items list - items[:] = custom_logger_tests + other_tests diff --git a/tests/vector_store_tests/rag/base_rag_tests.py b/tests/vector_store_tests/rag/base_rag_tests.py deleted file mode 100644 index 70a78144b80..00000000000 --- a/tests/vector_store_tests/rag/base_rag_tests.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -Base RAG test class that enforces common tests across all providers. - -Providers should inherit from BaseRAGTest and implement the abstract methods. -""" - -import uuid -from abc import ABC, abstractmethod -from typing import Any, Dict, Optional - -import pytest - - -import litellm -from litellm.types.rag import ( - RAGIngestOptions, - OpenAIVectorStoreOptions, - BedrockVectorStoreOptions, -) - - -class BaseRAGTest(ABC): - """ - Abstract base test class for RAG ingestion tests. - - Providers should inherit from this class and implement: - - get_base_ingest_options(): Returns provider-specific ingest options - - query_vector_store(): Queries the vector store after ingestion - """ - - @abstractmethod - def get_base_ingest_options(self) -> RAGIngestOptions: - """ - Must return the base ingest options for the provider. - - Example for OpenAI: - return { - "vector_store": OpenAIVectorStoreOptions( - custom_llm_provider="openai", - ) - } - - Example for Bedrock: - return { - "vector_store": BedrockVectorStoreOptions( - custom_llm_provider="bedrock", - ) - } - """ - pass - - @abstractmethod - async def query_vector_store( - self, - vector_store_id: str, - query: str, - ) -> Optional[Dict[str, Any]]: - """ - Query the vector store to verify ingestion. - - Args: - vector_store_id: The ID of the vector store to query - query: The search query - - Returns: - Search results dict or None if no results found - """ - pass - - def get_unique_filename(self, prefix: str = "test") -> str: - """Generate a unique filename for test documents.""" - unique_id = uuid.uuid4().hex[:8] - return f"{prefix}_{unique_id}.txt", unique_id - - @pytest.mark.asyncio - async def test_basic_ingest(self): - """ - Test basic text file ingestion to vector store. - """ - litellm.turn_on_debug() - - filename, unique_id = self.get_unique_filename("basic_ingest") - text_content = f"Test document {unique_id} for RAG ingestion.".encode("utf-8") - file_data = (filename, text_content, "text/plain") - - ingest_options = self.get_base_ingest_options() - ingest_options["name"] = f"test-basic-ingest-{unique_id}" - - try: - response = await litellm.rag.aingest( - ingest_options=ingest_options, - file_data=file_data, - ) - - print(f"RAG Ingest Response: {response}") - - assert "id" in response - assert response["id"].startswith("ingest_") - assert "status" in response - assert response["status"] in ["completed", "failed"] - assert "vector_store_id" in response - - if response["status"] == "completed": - assert response["vector_store_id"] - print(f"Vector store ID: {response['vector_store_id']}") - - except litellm.InternalServerError: - pytest.skip("Skipping test due to litellm.InternalServerError") - - @pytest.mark.asyncio - async def test_ingest_and_query(self): - """ - Test full RAG flow: ingest a document and then query it. - """ - import asyncio - - litellm.turn_on_debug() - - filename, unique_id = self.get_unique_filename("ingest_query") - text_content = f""" - Test document {unique_id} for RAG ingestion and query. - LiteLLM provides a unified interface for 100+ LLMs. - This content should be retrievable via semantic search. - """.encode( - "utf-8" - ) - file_data = (filename, text_content, "text/plain") - - ingest_options = self.get_base_ingest_options() - ingest_options["name"] = f"test-ingest-query-{unique_id}" - - try: - # Step 1: Ingest - ingest_response = await litellm.rag.aingest( - ingest_options=ingest_options, - file_data=file_data, - ) - - print(f"Ingest Response: {ingest_response}") - assert ingest_response["status"] == "completed" - vector_store_id = ingest_response["vector_store_id"] - assert vector_store_id - - # Step 2: Query with retry (indexing may take time) - search_results = None - max_retries = 10 - for attempt in range(max_retries): - await asyncio.sleep(3) - - search_results = await self.query_vector_store( - vector_store_id=vector_store_id, - query=f"Test document {unique_id}", - ) - - if search_results: - break - - print( - f"Attempt {attempt + 1}/{max_retries}: " - "Waiting for document to be indexed..." - ) - - print(f"Search Results: {search_results}") - - # Validate search results - assert search_results is not None, "Document not found after retries" - - print("Query successful!") - - except litellm.InternalServerError: - pytest.skip("Skipping test due to litellm.InternalServerError") diff --git a/tests/vector_store_tests/rag/test_document.txt b/tests/vector_store_tests/rag/test_document.txt deleted file mode 100644 index 3570e715e83..00000000000 --- a/tests/vector_store_tests/rag/test_document.txt +++ /dev/null @@ -1,4 +0,0 @@ -Test document abc123 for RAG ingestion. -This is a sample document to test the RAG ingest API. -LiteLLM provides a unified interface for vector stores. - diff --git a/tests/vector_store_tests/rag/test_rag_bedrock.py b/tests/vector_store_tests/rag/test_rag_bedrock.py deleted file mode 100644 index 90cf4a3a44e..00000000000 --- a/tests/vector_store_tests/rag/test_rag_bedrock.py +++ /dev/null @@ -1,88 +0,0 @@ -""" -Bedrock Knowledge Base RAG ingestion tests. - -Requires environment variables: -- AWS_ACCESS_KEY_ID -- AWS_SECRET_ACCESS_KEY -- AWS_REGION_NAME (optional, defaults to us-west-2) - -Optional (for using existing KB instead of auto-creating): -- BEDROCK_KNOWLEDGE_BASE_ID -""" - -import os -from typing import Any, Dict, Optional - -import pytest - - -import litellm -from litellm.types.rag import RAGIngestOptions, BedrockVectorStoreOptions -from tests.vector_store_tests.rag.base_rag_tests import BaseRAGTest - - -class TestRAGBedrock(BaseRAGTest): - """Test RAG Ingest with Bedrock Knowledge Base.""" - - @pytest.fixture(autouse=True) - def check_env_vars(self): - """Check required environment variables before each test.""" - aws_key = os.environ.get("AWS_ACCESS_KEY_ID") - aws_secret = os.environ.get("AWS_SECRET_ACCESS_KEY") - - if not aws_key or not aws_secret: - pytest.skip("Skipping Bedrock test: AWS credentials required") - - def get_base_ingest_options(self) -> RAGIngestOptions: - """ - Return Bedrock-specific ingest options. - - Uses unified interface - no vector_store_id means auto-create KB. - If BEDROCK_KNOWLEDGE_BASE_ID is set, uses existing KB. - """ - # Use existing KB if provided, otherwise auto-create - existing_kb_id = os.environ.get("BEDROCK_KNOWLEDGE_BASE_ID") - - return { - "vector_store": BedrockVectorStoreOptions( - custom_llm_provider="bedrock", - vector_store_id=existing_kb_id, # None = auto-create - # wait_for_ingestion defaults to False - returns immediately - ), - } - - async def query_vector_store( - self, - vector_store_id: str, - query: str, - ) -> Optional[Dict[str, Any]]: - """Query Bedrock Knowledge Base.""" - try: - import boto3 - except ImportError: - pytest.skip("boto3 required for Bedrock tests") - - session = boto3.Session( - aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"), - aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"), - region_name=os.environ.get("AWS_REGION_NAME", "us-west-2"), - ) - bedrock_agent_runtime = session.client("bedrock-agent-runtime") - - response = bedrock_agent_runtime.retrieve( - knowledgeBaseId=vector_store_id, - retrievalQuery={"text": query}, - retrievalConfiguration={ - "vectorSearchConfiguration": {"numberOfResults": 5} - }, - ) - - if response.get("retrievalResults") and len(response["retrievalResults"]) > 0: - # Check if query terms appear in results - for result in response["retrievalResults"]: - # Extract unique_id from query if present - if query in result["content"]["text"]: - return response - # Return results even if exact match not found - return response - return None diff --git a/tests/vector_store_tests/rag/test_rag_openai.py b/tests/vector_store_tests/rag/test_rag_openai.py deleted file mode 100644 index f90519a17e6..00000000000 --- a/tests/vector_store_tests/rag/test_rag_openai.py +++ /dev/null @@ -1,140 +0,0 @@ -""" -OpenAI RAG ingestion tests. -""" - -from typing import Any, Dict, Optional - -import pytest - - -import litellm -from litellm.types.rag import RAGIngestOptions, OpenAIVectorStoreOptions -from tests.vector_store_tests.rag.base_rag_tests import BaseRAGTest - - -class TestRAGOpenAI(BaseRAGTest): - """Test RAG Ingest with OpenAI provider.""" - - def get_base_ingest_options(self) -> RAGIngestOptions: - """Return OpenAI-specific ingest options.""" - return { - "vector_store": OpenAIVectorStoreOptions( - custom_llm_provider="openai", - ), - } - - async def query_vector_store( - self, - vector_store_id: str, - query: str, - ) -> Optional[Dict[str, Any]]: - """Query OpenAI vector store.""" - search_response = await litellm.vector_stores.asearch( - vector_store_id=vector_store_id, - query=query, - custom_llm_provider="openai", - ) - - if search_response.get("data") and len(search_response["data"]) > 0: - return search_response - return None - - @pytest.mark.asyncio - async def test_rag_query_basic(self): - """Test basic RAG query flow.""" - import asyncio - - litellm.turn_on_debug() - - # First ingest a document - filename, unique_id = self.get_unique_filename("rag_query") - text_content = ( - f"LiteLLM is a unified interface for 100+ LLMs. ID: {unique_id}".encode() - ) - - ingest_response = await litellm.rag.aingest( - ingest_options=self.get_base_ingest_options(), - file_data=(filename, text_content, "text/plain"), - ) - - # Check if ingestion succeeded - if ingest_response["status"] != "completed": - pytest.fail( - f"Ingestion failed with status: {ingest_response['status']}, " - f"error: {ingest_response.get('error', 'Unknown')}" - ) - - vector_store_id = ingest_response["vector_store_id"] - assert vector_store_id, "vector_store_id should not be empty" - - # Wait for indexing - await asyncio.sleep(10) - - # Query with RAG - response = await litellm.rag.aquery( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "What is LiteLLM?"}], - retrieval_config={ - "vector_store_id": vector_store_id, - "custom_llm_provider": "openai", - "top_k": 5, - }, - ) - - print(f"RAG Query Response: {response}") - - assert response.choices[0].message.content - assert "search_results" in response.choices[0].message.provider_specific_fields - - @pytest.mark.asyncio - async def test_rag_query_with_rerank(self): - """Test RAG query with reranking.""" - import asyncio - - litellm.turn_on_debug() - - # First ingest a document - filename, unique_id = self.get_unique_filename("rag_query_rerank") - text_content = ( - f"LiteLLM is a unified interface for 100+ LLMs. ID: {unique_id}".encode() - ) - - ingest_response = await litellm.rag.aingest( - ingest_options=self.get_base_ingest_options(), - file_data=(filename, text_content, "text/plain"), - ) - - # Check if ingestion succeeded - if ingest_response["status"] != "completed": - pytest.fail( - f"Ingestion failed with status: {ingest_response['status']}, " - f"error: {ingest_response.get('error', 'Unknown')}" - ) - - vector_store_id = ingest_response["vector_store_id"] - assert vector_store_id, "vector_store_id should not be empty" - - # Wait for indexing - await asyncio.sleep(10) - - # Query with RAG and rerank - response = await litellm.rag.aquery( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "What is LiteLLM?"}], - retrieval_config={ - "vector_store_id": vector_store_id, - "custom_llm_provider": "openai", - "top_k": 5, - }, - rerank={ - "enabled": True, - "model": "cohere/rerank-english-v3.0", - "top_n": 3, - }, - ) - - print(f"RAG Query Response with Rerank: {response.model_dump_json(indent=4)}") - - assert response.choices[0].message.content - assert "search_results" in response.choices[0].message.provider_specific_fields - assert "rerank_results" in response.choices[0].message.provider_specific_fields diff --git a/tests/vector_store_tests/rag/test_rag_s3_vectors.py b/tests/vector_store_tests/rag/test_rag_s3_vectors.py deleted file mode 100644 index d950bc0f644..00000000000 --- a/tests/vector_store_tests/rag/test_rag_s3_vectors.py +++ /dev/null @@ -1,105 +0,0 @@ -""" -S3 Vectors RAG ingestion tests. - -Requires environment variables: -- AWS_ACCESS_KEY_ID -- AWS_SECRET_ACCESS_KEY -- AWS_REGION_NAME (optional, defaults to us-west-2) - -Optional: -- S3_VECTOR_BUCKET_NAME (optional, auto-generates if not set) -""" - -import os -from typing import Any, Dict, Optional - -import pytest - - -import litellm -from litellm.types.rag import RAGIngestOptions -from tests.vector_store_tests.rag.base_rag_tests import BaseRAGTest - - -class TestRAGS3Vectors(BaseRAGTest): - """Test RAG Ingest with AWS S3 Vectors.""" - - @pytest.fixture(autouse=True) - def check_env_vars(self): - """Check required environment variables before each test.""" - aws_key = os.environ.get("AWS_ACCESS_KEY_ID") - aws_secret = os.environ.get("AWS_SECRET_ACCESS_KEY") - - if not aws_key or not aws_secret: - pytest.skip("Skipping S3 Vectors test: AWS credentials required") - - def get_base_ingest_options(self) -> RAGIngestOptions: - """ - Return S3 Vectors-specific ingest options. - - Chunking is configured via chunking_strategy (unified interface). - Embeddings are generated using LiteLLM's embedding API. - """ - vector_bucket_name = os.environ.get( - "S3_VECTOR_BUCKET_NAME", "test-litellm-vectors" - ) - aws_region = os.environ.get("AWS_REGION_NAME", "us-west-2") - - return { - "chunking_strategy": { - "chunk_size": 512, - "chunk_overlap": 100, - }, - "embedding": { - "model": "text-embedding-3-small" # Can use any LiteLLM-supported model - }, - "vector_store": { - "custom_llm_provider": "s3_vectors", - "vector_bucket_name": vector_bucket_name, - "index_name": "test-index", - # dimension is auto-detected from embedding model (text-embedding-3-small = 1536) - "distance_metric": "cosine", - "non_filterable_metadata_keys": ["source_text"], - "aws_region_name": aws_region, - }, - } - - async def query_vector_store( - self, - vector_store_id: str, - query: str, - ) -> Optional[Dict[str, Any]]: - """Query S3 Vectors index.""" - try: - # Import the ingestion class to use its query method - from litellm.rag.ingestion.s3_vectors_ingestion import ( - S3VectorsRAGIngestion, - ) - except ImportError: - pytest.skip("S3 Vectors ingestion not available") - - vector_bucket_name = os.environ.get( - "S3_VECTOR_BUCKET_NAME", "test-litellm-vectors" - ) - aws_region = os.environ.get("AWS_REGION_NAME", "us-west-2") - - # Create ingestion instance to use query method - ingest_options = { - "embedding": {"model": "text-embedding-3-small"}, - "vector_store": { - "custom_llm_provider": "s3_vectors", - "vector_bucket_name": vector_bucket_name, - "aws_region_name": aws_region, - }, - } - - ingestion = S3VectorsRAGIngestion(ingest_options=ingest_options) - - # Query the index - results = await ingestion.query_vector_store( - vector_store_id=vector_store_id, - query=query, - top_k=5, - ) - - return results diff --git a/tests/vector_store_tests/rag/test_rag_vertex_ai.py b/tests/vector_store_tests/rag/test_rag_vertex_ai.py deleted file mode 100644 index 67e0a25782b..00000000000 --- a/tests/vector_store_tests/rag/test_rag_vertex_ai.py +++ /dev/null @@ -1,243 +0,0 @@ -""" -Vertex AI RAG Engine ingestion tests. - -Tests the Vertex AI RAG ingestion implementation that: -- Creates RAG corpora automatically (or uses existing ones) -- Uploads files directly to Vertex AI RAG Engine -- Handles long-running operations for corpus creation -- Supports both file upload and GCS import - -Requires: -- gcloud auth application-default login (for ADC authentication) - -Environment variables: -- VERTEX_PROJECT: GCP project ID (required) -- VERTEX_LOCATION: GCP region (optional, defaults to us-central1) -- VERTEX_CORPUS_ID: Existing RAG corpus ID (optional - will create if not provided) -""" - -import os -from typing import Any, Dict, Optional - -import pytest - - -import litellm -from litellm.types.rag import RAGIngestOptions -from tests.vector_store_tests.rag.base_rag_tests import BaseRAGTest - - -class TestRAGVertexAI(BaseRAGTest): - """Test RAG Ingest with Vertex AI RAG Engine.""" - - @pytest.fixture(autouse=True) - def check_env_vars(self): - """Check required environment variables before each test.""" - vertex_project = os.environ.get("VERTEX_PROJECT") - - if not vertex_project: - pytest.skip("Skipping Vertex AI test: VERTEX_PROJECT required") - - def get_base_ingest_options(self) -> RAGIngestOptions: - """ - Return Vertex AI-specific ingest options. - - Chunking is configured via chunking_strategy (unified interface), - not inside vector_store. - - If VERTEX_CORPUS_ID is not set, a new corpus will be created automatically. - """ - vertex_project = os.environ.get("VERTEX_PROJECT") - vertex_location = os.environ.get("VERTEX_LOCATION", "us-central1") - corpus_id = os.environ.get("VERTEX_CORPUS_ID") # Optional - - options: RAGIngestOptions = { - "chunking_strategy": { - "chunk_size": 512, - "chunk_overlap": 100, - }, - "vector_store": { - "custom_llm_provider": "vertex_ai", - "vertex_project": vertex_project, - "vertex_location": vertex_location, - }, - } - - # Add corpus ID if provided (otherwise will create new corpus) - if corpus_id: - options["vector_store"]["vector_store_id"] = corpus_id - - return options - - async def query_vector_store( - self, - vector_store_id: str, - query: str, - ) -> Optional[Dict[str, Any]]: - """ - Query Vertex AI RAG corpus using LiteLLM's vector store search. - - Args: - vector_store_id: The RAG corpus ID (can be full path or just the ID) - query: The search query - - Returns: - Search results dict or None if no results found - """ - vertex_project = os.environ.get("VERTEX_PROJECT") - vertex_location = os.environ.get("VERTEX_LOCATION", "us-central1") - - try: - # Use LiteLLM's vector store search - search_response = await litellm.vector_stores.asearch( - vector_store_id=vector_store_id, - query=query, - max_num_results=5, - custom_llm_provider="vertex_ai", - vertex_project=vertex_project, - vertex_location=vertex_location, - ) - - # Check if we got results - if search_response and search_response.get("data"): - results = [] - for item in search_response["data"]: - # Extract text from content - text = "" - if item.get("content"): - for content_item in item["content"]: - if content_item.get("text"): - text += content_item["text"] - - results.append( - { - "text": text, - "score": item.get("score", 0.0), - "file_id": item.get("file_id", ""), - "filename": item.get("filename", ""), - } - ) - - # Check if query terms appear in results - for result in results: - if query.lower() in result["text"].lower(): - return {"results": results} - - # Return results even if exact match not found - return {"results": results} - - return None - - except Exception as e: - print(f"Query failed: {e}") - return None - - @pytest.mark.asyncio - async def test_create_corpus_and_ingest(self): - """ - Test creating a new RAG corpus and ingesting a file. - - This test specifically validates: - - Automatic corpus creation when vector_store_id is not provided - - Long-running operation polling for corpus creation - - File upload to the newly created corpus - """ - litellm.turn_on_debug() - - filename, unique_id = self.get_unique_filename("create_corpus") - text_content = f""" - Test document {unique_id} for Vertex AI RAG corpus creation. - This tests the automatic corpus creation feature. - The corpus should be created and the file should be uploaded successfully. - """.encode( - "utf-8" - ) - file_data = (filename, text_content, "text/plain") - - # Get base options WITHOUT corpus_id to trigger creation - ingest_options = self.get_base_ingest_options() - # Remove corpus_id if it was set from env var - if "vector_store_id" in ingest_options.get("vector_store", {}): - del ingest_options["vector_store"]["vector_store_id"] - - ingest_options["name"] = f"test-create-corpus-{unique_id}" - - try: - response = await litellm.rag.aingest( - ingest_options=ingest_options, - file_data=file_data, - ) - - print(f"Create Corpus Response: {response}") - - # Validate response - assert "id" in response - assert response["id"].startswith("ingest_") - assert "status" in response - assert ( - response["status"] == "completed" - ), f"Expected completed, got {response['status']}" - assert "vector_store_id" in response - assert response["vector_store_id"], "vector_store_id should not be empty" - - # The vector_store_id should be a full corpus path - corpus_id = response["vector_store_id"] - assert "projects/" in corpus_id, "Corpus ID should be a full resource path" - assert "ragCorpora/" in corpus_id, "Corpus ID should contain ragCorpora" - - print(f"✓ Successfully created corpus: {corpus_id}") - print(f"✓ Successfully uploaded file: {response.get('file_id')}") - - except litellm.InternalServerError as e: - pytest.skip(f"Skipping test due to litellm.InternalServerError: {e}") - except Exception as e: - print(f"Test failed with error: {e}") - raise - - @pytest.mark.asyncio - async def test_ingest_with_existing_corpus(self): - """ - Test ingesting a file to an existing RAG corpus. - - This test validates: - - Using an existing corpus_id from environment variable - - Direct file upload without corpus creation - """ - corpus_id = os.environ.get("VERTEX_CORPUS_ID") - if not corpus_id: - pytest.skip("Skipping test: VERTEX_CORPUS_ID not set") - - litellm.turn_on_debug() - - filename, unique_id = self.get_unique_filename("existing_corpus") - text_content = f""" - Test document {unique_id} for existing Vertex AI RAG corpus. - This tests file upload to a pre-existing corpus. - """.encode( - "utf-8" - ) - file_data = (filename, text_content, "text/plain") - - ingest_options = self.get_base_ingest_options() - ingest_options["name"] = f"test-existing-corpus-{unique_id}" - - try: - response = await litellm.rag.aingest( - ingest_options=ingest_options, - file_data=file_data, - ) - - print(f"Existing Corpus Ingest Response: {response}") - - assert response["status"] == "completed" - assert ( - response["vector_store_id"] == corpus_id - or corpus_id in response["vector_store_id"] - ) - assert response.get("file_id"), "file_id should be present" - - print(f"✓ Successfully uploaded to existing corpus: {corpus_id}") - print(f"✓ File ID: {response.get('file_id')}") - - except litellm.InternalServerError as e: - pytest.skip(f"Skipping test due to litellm.InternalServerError: {e}") diff --git a/tests/vector_store_tests/test_azure_vector_store.py b/tests/vector_store_tests/test_azure_vector_store.py deleted file mode 100644 index 851492474cc..00000000000 --- a/tests/vector_store_tests/test_azure_vector_store.py +++ /dev/null @@ -1,25 +0,0 @@ -from base_vector_store_test import BaseVectorStoreTest -import os -import pytest - - -class TestAzureOpenAIVectorStore(BaseVectorStoreTest): - def get_base_request_args(self) -> dict: - """Must return the base request args""" - return {} - - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - async def test_basic_search_vector_store(self, sync_mode): - pass - - def get_base_create_vector_store_args(self) -> dict: - """ - This is a real vector store on Azure - """ - return { - "custom_llm_provider": "azure", - "api_base": os.getenv("AZURE_API_BASE"), - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": "2025-04-01-preview", - } diff --git a/tests/vector_store_tests/test_bedrock_vector_store.py b/tests/vector_store_tests/test_bedrock_vector_store.py deleted file mode 100644 index d8af1c7188b..00000000000 --- a/tests/vector_store_tests/test_bedrock_vector_store.py +++ /dev/null @@ -1,236 +0,0 @@ -""" -Test Bedrock Vector Store helper functions and transformation. -""" - -import pytest -from unittest.mock import Mock -import httpx - -from tests.vector_store_tests.base_vector_store_test import BaseVectorStoreTest -from litellm.llms.bedrock.vector_stores.transformation import BedrockVectorStoreConfig -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - - -class TestBedrockVectorStore(BaseVectorStoreTest): - """ - Test the Bedrock vector store transformation functionality. - """ - - def get_base_create_vector_store_args(self) -> dict: - """Must return the base create vector store args""" - return {} - - def get_base_request_args(self): - return { - "vector_store_id": "T37J8R4WTM", - "custom_llm_provider": "bedrock", - "query": "what happens after we add a model", - } - - def test_get_file_id_from_metadata(self): - """Test that file_id is correctly extracted from metadata.""" - config = BedrockVectorStoreConfig() - - # Test with source URI - metadata_with_uri = { - "x-amz-bedrock-kb-source-uri": "https://www.litellm.ai", - "x-amz-bedrock-kb-chunk-id": "1%3A0%3AjNYPg5YByRuP5PdK96co", - } - file_id = config._get_file_id_from_metadata(metadata_with_uri) - assert file_id == "https://www.litellm.ai" - - # Test without source URI but with chunk ID - metadata_without_uri = { - "x-amz-bedrock-kb-chunk-id": "1%3A0%3AjNYPg5YByRuP5PdK96co" - } - file_id = config._get_file_id_from_metadata(metadata_without_uri) - assert file_id == "bedrock-kb-1%3A0%3AjNYPg5YByRuP5PdK96co" - - # Test with empty metadata - file_id = config._get_file_id_from_metadata({}) - assert file_id == "bedrock-kb-unknown" - - def test_get_filename_from_metadata(self): - """Test that filename is correctly extracted from metadata.""" - config = BedrockVectorStoreConfig() - - # Test with source URI containing path - metadata_with_path = { - "x-amz-bedrock-kb-source-uri": "https://docs.litellm.ai/tutorial/setup.html" - } - filename = config._get_filename_from_metadata(metadata_with_path) - assert filename == "setup.html" - - # Test with source URI without path (domain only) - metadata_domain_only = {"x-amz-bedrock-kb-source-uri": "https://www.litellm.ai"} - filename = config._get_filename_from_metadata(metadata_domain_only) - assert filename == "www.litellm.ai" - - # Test without source URI but with data source ID - metadata_without_uri = {"x-amz-bedrock-kb-data-source-id": "CCEJIRXXFI"} - filename = config._get_filename_from_metadata(metadata_without_uri) - assert filename == "bedrock-kb-document-CCEJIRXXFI" - - # Test with empty metadata - filename = config._get_filename_from_metadata({}) - assert filename == "bedrock-kb-document-unknown" - - def test_get_attributes_from_metadata(self): - """Test that attributes are correctly extracted from metadata.""" - config = BedrockVectorStoreConfig() - - # Test with full metadata - metadata = { - "x-amz-bedrock-kb-source-uri": "https://www.litellm.ai", - "x-amz-bedrock-kb-chunk-id": "1%3A0%3AjNYPg5YByRuP5PdK96co", - "x-amz-bedrock-kb-data-source-id": "CCEJIRXXFI", - } - attributes = config._get_attributes_from_metadata(metadata) - assert attributes == metadata - assert attributes is not metadata # Should be a copy - - # Test with empty metadata - attributes = config._get_attributes_from_metadata({}) - assert attributes == {} - - # Test with None - attributes = config._get_attributes_from_metadata(None) - assert attributes == {} - - -@pytest.mark.asyncio -async def test_bedrock_search_with_router(): - from litellm.router import Router - - # init router - _router = Router(model_list=[]) - search_response = await _router.avector_store_search( - query="what happens after we add a model", - vector_store_id="T37J8R4WTM", - custom_llm_provider="bedrock", - ) - print(search_response) - - -@pytest.mark.asyncio -async def test_bedrock_search_with_credentials_managed_registry(): - """ - Test that the vector store search uses the credential accessor from the registry - when AWS environment variables are not set, ensuring credentials are managed properly. - """ - from unittest.mock import patch, MagicMock - from litellm.router import Router - from litellm.types.vector_stores import LiteLLM_ManagedVectorStore - from litellm.types.utils import CredentialItem - from litellm.vector_stores.vector_store_registry import VectorStoreRegistry - from datetime import datetime, timezone - import litellm - - # Store original registry and credential list - original_registry = getattr(litellm, "vector_store_registry", None) - original_credential_list = getattr(litellm, "credential_list", []) - - try: - # Set up test AWS credentials in the credential system - test_credentials = CredentialItem( - credential_name="bedrock-litellm-website-knowledgebase", - credential_info={ - "provider": "aws", - "description": "Test AWS credentials for bedrock", - }, - credential_values={ - "aws_access_key_id": "test_access_key", - "aws_secret_access_key": "test_secret_key", - "aws_region_name": "us-east-1", - }, - ) - - # Set up the credential list - litellm.credential_list = [test_credentials] - - # Create vector store with credential reference - vector_store = LiteLLM_ManagedVectorStore( - vector_store_id="T37J8R4WTM", - custom_llm_provider="bedrock", - created_at=datetime.now(timezone.utc), - updated_at=datetime.now(timezone.utc), - litellm_credential_name="bedrock-litellm-website-knowledgebase", - ) - - # Set up registry - registry = VectorStoreRegistry([vector_store]) - litellm.vector_store_registry = registry - - # Verify credentials can be retrieved from registry - retrieved_credentials = registry.get_credentials_for_vector_store("T37J8R4WTM") - assert retrieved_credentials, "Should retrieve credentials from registry" - assert retrieved_credentials.get("aws_access_key_id") == "test_access_key" - assert retrieved_credentials.get("aws_secret_access_key") == "test_secret_key" - assert retrieved_credentials.get("aws_region_name") == "us-east-1" - - # Create router and perform search - _router = Router(model_list=[]) - - # Mock the credential injection process to verify it's called - with patch.object( - registry, - "get_credentials_for_vector_store", - wraps=registry.get_credentials_for_vector_store, - ) as mock_get_creds: - # Mock the actual search call to avoid making real API calls - with patch( - "litellm.vector_stores.main.base_llm_http_handler.vector_store_search_handler" - ) as mock_handler: - mock_handler.return_value = { - "data": [ - { - "id": "test_result", - "text": "Mock search result", - "score": 0.9, - "metadata": {}, - } - ] - } - - search_response = await _router.avector_store_search( - query="what happens after we add a model", - vector_store_id="T37J8R4WTM", - custom_llm_provider="bedrock", - ) - - # Verify the search was called - mock_handler.assert_called_once() - call_kwargs = mock_handler.call_args[1] - - # Verify that the credential accessor was called with the correct vector store ID - mock_get_creds.assert_called_with("T37J8R4WTM") - - # Verify the credentials were injected into the search call - litellm_params = call_kwargs.get("litellm_params", {}) - - # The key test: verify that credentials from the registry were used - # Since we have a registry with credentials, they should be present in the params - assert hasattr( - litellm_params, "aws_access_key_id" - ), "aws_access_key_id should be in litellm_params" - assert hasattr( - litellm_params, "aws_secret_access_key" - ), "aws_secret_access_key should be in litellm_params" - assert hasattr( - litellm_params, "aws_region_name" - ), "aws_region_name should be in litellm_params" - - # Verify we got the expected response - assert search_response["data"][0]["id"] == "test_result" - - print( - f"✅ Test passed: Credential accessor was called with vector store ID: T37J8R4WTM" - ) - print(f"✅ Retrieved credentials: {retrieved_credentials}") - print(f"✅ Credentials were injected into search call") - print(f"✅ Search completed successfully using registry credentials") - - finally: - # Restore original state - litellm.vector_store_registry = original_registry - litellm.credential_list = original_credential_list diff --git a/tests/vector_store_tests/test_gemini_vector_store.py b/tests/vector_store_tests/test_gemini_vector_store.py deleted file mode 100644 index 2aa2c1741a8..00000000000 --- a/tests/vector_store_tests/test_gemini_vector_store.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -Minimal Gemini File Search vector store tests. -""" - -import os - - -from base_vector_store_test import BaseVectorStoreTest - - -class TestGeminiVectorStore(BaseVectorStoreTest): - """Reuses the shared vector store smoke suite with Gemini.""" - - def get_base_request_args(self) -> dict: - """Provide arguments for the shared search test.""" - return { - "vector_store_id": os.getenv( - "GEMINI_TEST_STORE_ID", "fileSearchStores/example-test-store" - ), - "custom_llm_provider": "gemini", - "query": "LiteLLM", - } - - def get_base_create_vector_store_args(self) -> dict: - """Ensure we always call Gemini when creating a vector store.""" - return { - "custom_llm_provider": "gemini", - } diff --git a/tests/vector_store_tests/test_milvus_vector_store.py b/tests/vector_store_tests/test_milvus_vector_store.py deleted file mode 100644 index 2139abb6a90..00000000000 --- a/tests/vector_store_tests/test_milvus_vector_store.py +++ /dev/null @@ -1,717 +0,0 @@ -""" -Tests for Milvus Vector Store -""" - -import json -from unittest.mock import AsyncMock, MagicMock, patch - -import httpx -import pytest -import respx - -import litellm -from litellm import Router -from litellm.llms.milvus.vector_stores.transformation import MilvusVectorStoreConfig -from litellm.types.utils import EmbeddingResponse -from litellm.vector_stores import asearch as vector_store_asearch -from litellm.vector_stores import search as vector_store_search - -# Mock response from actual Milvus API -MOCK_MILVUS_SEARCH_RESPONSE = { - "code": 0, - "cost": 6, - "data": [ - { - "book_id": 0, - "book_intro_text": "abababababa_0562efee-0f1f-4b6b-9ca3-1a160f124ad8", - "distance": 10.240219, - }, - { - "book_id": 1, - "book_intro_text": "abababababa_9a13e8f3-bb1e-487f-b555-b8ae4b127243", - "distance": 10.240219, - }, - { - "book_id": 2, - "book_intro_text": "abababababa_870f47f1-23ec-4364-ad30-6d364ba8ddb5", - "distance": 10.240219, - }, - { - "book_id": 1000, - "book_intro_text": "abababababa_8ea2d76a-3fdf-49b3-8f16-a91638361bba", - "distance": 8.531628, - }, - { - "book_id": 1001, - "book_intro_text": "abababababa_24758251-e740-4183-8649-2f742f676ca0", - "distance": 8.531628, - }, - { - "book_id": 1002, - "book_intro_text": "abababababa_faa55789-220d-4ef1-b5bf-a72f2fbd061b", - "distance": 8.531628, - }, - { - "book_id": 0, - "book_intro_text": "abababababa_0562efee-0f1f-4b6b-9ca3-1a160f124ad8", - "distance": 8.236887, - }, - { - "book_id": 1, - "book_intro_text": "abababababa_9a13e8f3-bb1e-487f-b555-b8ae4b127243", - "distance": 8.236887, - }, - { - "book_id": 2, - "book_intro_text": "abababababa_870f47f1-23ec-4364-ad30-6d364ba8ddb5", - "distance": 8.236887, - }, - ], - "topks": [3, 3, 3], -} -# Mock embedding response from OpenAI -MOCK_EMBEDDING_RESPONSE = MagicMock() -MOCK_EMBEDDING_RESPONSE.data = [ - { - "embedding": [ - 0.023, - -0.019, - 0.045, - -0.012, - 0.067, - -0.034, - 0.089, - -0.056, - ] - * 128 # Simulate 1024-dimensional embedding - } -] - - -class TestMilvusVectorStore: - """Test Milvus Vector Store with mocked responses""" - - @pytest.mark.asyncio - async def test_basic_search_with_mock_async(self): - """Test basic vector search with mocked backend response (async)""" - - # Mock the HTTP response - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = MOCK_MILVUS_SEARCH_RESPONSE - mock_response.text = json.dumps(MOCK_MILVUS_SEARCH_RESPONSE) - - with patch("litellm.aembedding", new_callable=AsyncMock) as mock_embedding: - mock_embedding.return_value = MOCK_EMBEDDING_RESPONSE - - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - new_callable=AsyncMock, - ) as mock_post: - mock_post.return_value = mock_response - - # Make the search request - response = await vector_store_asearch( - query="what is machine learning?", - vector_store_id="book_2", - custom_llm_provider="milvus", - api_base="https://in03-test.serverless.aws-eu-central-1.cloud.zilliz.com", - api_key="mock_milvus_api_key", - litellm_embedding_model="text-embedding-3-large", - litellm_embedding_config={ - "api_key": "mock_openai_api_key", - }, - outputFields=["book_intro_text"], - annsField="book_intro_vector", - milvus_text_field="book_intro_text", - ) - - print("Response:", json.dumps(response, indent=2, default=str)) - - # Verify embedding was called with correct parameters - mock_embedding.assert_called_once() - embedding_call_args = mock_embedding.call_args - assert embedding_call_args[1]["model"] == "text-embedding-3-large" - assert embedding_call_args[1]["input"] == ["what is machine learning?"] - assert embedding_call_args[1]["api_key"] == "mock_openai_api_key" - - # Verify the API was called - mock_post.assert_called_once() - - # Verify the request payload - call_args = mock_post.call_args - print(f"call_args: {call_args}") - print(f"call_args.kwargs: {call_args.kwargs}") - - # The post method is called with 'data' parameter (JSON string) not 'json' parameter - request_data_str = call_args.kwargs.get("data") - if request_data_str: - request_data = json.loads(request_data_str) - else: - # Fallback: check for json kwarg or in args - request_data = call_args.kwargs.get("json") - if request_data is None and len(call_args.args) > 0 and isinstance(call_args.args[0], dict): - request_data = call_args.args[0] - - assert request_data is not None, f"Could not extract request data. Call args: {call_args}" - print("Request data:", json.dumps(request_data, indent=2, default=str)) - - # Validate request structure - assert "collectionName" in request_data - assert request_data["collectionName"] == "book_2" - assert "data" in request_data - assert isinstance(request_data["data"], list) - assert len(request_data["data"]) == 1 # Single query vector - assert "annsField" in request_data - assert request_data["annsField"] == "book_intro_vector" - assert "outputFields" in request_data - assert request_data["outputFields"] == ["book_intro_text"] - - # Verify the URL format - url = call_args.kwargs.get("url", "") - assert "v2/vectordb/entities/search" in url - - # Validate the response structure (LiteLLM standard format) - assert response is not None - assert response["object"] == "vector_store.search_results.page" # type: ignore - assert "data" in response - assert len(response["data"]) == 9 # type: ignore # 9 results in mock response - - # Validate first result - first_result = response["data"][0] # type: ignore - assert "score" in first_result - assert first_result["score"] == 10.240219 # type: ignore - assert "content" in first_result - assert "attributes" in first_result - - # Validate content structure - assert len(first_result["content"]) > 0 # type: ignore - assert first_result["content"][0]["type"] == "text" # type: ignore - assert "text" in first_result["content"][0] # type: ignore - assert ( - first_result["content"][0]["text"] # type: ignore - == "abababababa_0562efee-0f1f-4b6b-9ca3-1a160f124ad8" - ) - - # Validate attributes contain book_id but NOT book_intro_text (it's in content) - assert "book_id" in first_result["attributes"] # type: ignore - assert first_result["attributes"]["book_id"] == 0 # type: ignore - assert "book_intro_text" not in first_result["attributes"] # type: ignore # Should be in content, not attributes - - def test_basic_search_with_mock_sync(self): - """Test basic vector search with mocked backend response (sync)""" - - # Mock the HTTP response - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = MOCK_MILVUS_SEARCH_RESPONSE - mock_response.text = json.dumps(MOCK_MILVUS_SEARCH_RESPONSE) - - with patch("litellm.embedding") as mock_embedding: - mock_embedding.return_value = MOCK_EMBEDDING_RESPONSE - - with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: - mock_post.return_value = mock_response - - # Make the search request - response = vector_store_search( - query="what is machine learning?", - vector_store_id="book_2", - custom_llm_provider="milvus", - api_base="https://in03-test.serverless.aws-eu-central-1.cloud.zilliz.com", - api_key="mock_milvus_api_key", - litellm_embedding_model="text-embedding-3-large", - litellm_embedding_config={ - "api_key": "mock_openai_api_key", - }, - outputFields=["book_intro_text"], - annsField="book_intro_vector", - milvus_text_field="book_intro_text", - ) - - print("Response:", json.dumps(response, indent=2, default=str)) - - # Verify embedding was called - mock_embedding.assert_called_once() - - # Verify the API was called - mock_post.assert_called_once() - - # Verify the request payload - call_args = mock_post.call_args - - # The post method is called with 'data' parameter (JSON string) not 'json' parameter - request_data_str = call_args.kwargs.get("data") - if request_data_str: - request_data = json.loads(request_data_str) - else: - # Fallback: check for json kwarg or in args - request_data = call_args.kwargs.get("json") - if request_data is None and len(call_args.args) > 0 and isinstance(call_args.args[0], dict): - request_data = call_args.args[0] - - assert request_data is not None, f"Could not extract request data. Call args: {call_args}" - - # Validate request structure - assert "collectionName" in request_data - assert request_data["collectionName"] == "book_2" - assert "data" in request_data - assert isinstance(request_data["data"], list) - assert "annsField" in request_data - assert "outputFields" in request_data - - # Validate the response structure - assert response is not None - assert response["object"] == "vector_store.search_results.page" # type: ignore - assert "data" in response # type: ignore - assert len(response["data"]) == 9 # type: ignore # 9 results in mock response - assert "search_query" in response # type: ignore - - # Validate first few results - expected_results = [ - { - "book_id": 0, - "text": "abababababa_0562efee-0f1f-4b6b-9ca3-1a160f124ad8", - "distance": 10.240219, - }, - { - "book_id": 1, - "text": "abababababa_9a13e8f3-bb1e-487f-b555-b8ae4b127243", - "distance": 10.240219, - }, - { - "book_id": 2, - "text": "abababababa_870f47f1-23ec-4364-ad30-6d364ba8ddb5", - "distance": 10.240219, - }, - ] - - for idx, expected in enumerate(expected_results): - result = response["data"][idx] # type: ignore - assert "score" in result - assert result["score"] == expected["distance"] # type: ignore - assert "content" in result - assert len(result["content"]) > 0 # type: ignore - assert result["content"][0]["type"] == "text" # type: ignore - assert "text" in result["content"][0] # type: ignore - assert result["content"][0]["text"] == expected["text"] # type: ignore - assert "attributes" in result - assert result["attributes"]["book_id"] == expected["book_id"] # type: ignore - assert "book_intro_text" not in result["attributes"] # type: ignore # Should be in content, not attributes - - def _extract_request_body(self, mock_post): - call_args = mock_post.call_args - request_data_str = call_args.kwargs.get("data") - if request_data_str: - return json.loads(request_data_str) - request_data = call_args.kwargs.get("json") - if request_data is None and len(call_args.args) > 0 and isinstance(call_args.args[0], dict): - request_data = call_args.args[0] - return request_data - - def test_user_supplied_db_and_partition_are_dropped(self): - """User-supplied dbName / partitionNames must not be forwarded to Milvus.""" - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = MOCK_MILVUS_SEARCH_RESPONSE - mock_response.text = json.dumps(MOCK_MILVUS_SEARCH_RESPONSE) - - with patch("litellm.embedding") as mock_embedding: - mock_embedding.return_value = MOCK_EMBEDDING_RESPONSE - - with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: - mock_post.return_value = mock_response - - vector_store_search( - query="what is machine learning?", - vector_store_id="book_2", - custom_llm_provider="milvus", - api_base="https://in03-test.serverless.aws-eu-central-1.cloud.zilliz.com", - api_key="mock_milvus_api_key", - litellm_embedding_model="text-embedding-3-large", - litellm_embedding_config={ - "api_key": "mock_openai_api_key", - }, - outputFields=["book_intro_text"], - annsField="book_intro_vector", - milvus_text_field="book_intro_text", - dbName="other_tenant_db", - partitionNames=["other_tenant_partition"], - ) - - mock_post.assert_called_once() - request_data = self._extract_request_body(mock_post) - assert request_data is not None - assert "dbName" not in request_data - assert "partitionNames" not in request_data - assert request_data["collectionName"] == "book_2" - assert request_data["annsField"] == "book_intro_vector" - assert request_data["outputFields"] == ["book_intro_text"] - - def test_backend_configured_db_and_partition_are_forwarded(self): - """milvus_db_name / milvus_partition_names from litellm_params must be sent.""" - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = MOCK_MILVUS_SEARCH_RESPONSE - mock_response.text = json.dumps(MOCK_MILVUS_SEARCH_RESPONSE) - - with patch("litellm.embedding") as mock_embedding: - mock_embedding.return_value = MOCK_EMBEDDING_RESPONSE - - with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: - mock_post.return_value = mock_response - - vector_store_search( - query="what is machine learning?", - vector_store_id="book_2", - custom_llm_provider="milvus", - api_base="https://in03-test.serverless.aws-eu-central-1.cloud.zilliz.com", - api_key="mock_milvus_api_key", - litellm_embedding_model="text-embedding-3-large", - litellm_embedding_config={ - "api_key": "mock_openai_api_key", - }, - outputFields=["book_intro_text"], - annsField="book_intro_vector", - milvus_text_field="book_intro_text", - milvus_db_name="tenant_a_db", - milvus_partition_names=["tenant_a_partition"], - ) - - mock_post.assert_called_once() - request_data = self._extract_request_body(mock_post) - assert request_data is not None - assert request_data["dbName"] == "tenant_a_db" - assert request_data["partitionNames"] == ["tenant_a_partition"] - - def test_user_params_cannot_override_backend_db_and_partition(self): - """Backend-config dbName/partitionNames must win over user-supplied values.""" - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = MOCK_MILVUS_SEARCH_RESPONSE - mock_response.text = json.dumps(MOCK_MILVUS_SEARCH_RESPONSE) - - with patch("litellm.embedding") as mock_embedding: - mock_embedding.return_value = MOCK_EMBEDDING_RESPONSE - - with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: - mock_post.return_value = mock_response - - vector_store_search( - query="what is machine learning?", - vector_store_id="book_2", - custom_llm_provider="milvus", - api_base="https://in03-test.serverless.aws-eu-central-1.cloud.zilliz.com", - api_key="mock_milvus_api_key", - litellm_embedding_model="text-embedding-3-large", - litellm_embedding_config={ - "api_key": "mock_openai_api_key", - }, - outputFields=["book_intro_text"], - annsField="book_intro_vector", - milvus_text_field="book_intro_text", - milvus_db_name="tenant_a_db", - milvus_partition_names=["tenant_a_partition"], - dbName="other_tenant_db", - partitionNames=["other_tenant_partition"], - ) - - mock_post.assert_called_once() - request_data = self._extract_request_body(mock_post) - assert request_data is not None - assert request_data["dbName"] == "tenant_a_db" - assert request_data["partitionNames"] == ["tenant_a_partition"] - - -# @pytest.mark.parametrize("sync_mode", [True, False]) -# @pytest.mark.asyncio -# async def test_basic_search_vector_store(sync_mode): -# """Integration test with real Milvus API (requires credentials)""" -# litellm.turn_on_debug() -# litellm.set_verbose = True -# base_request_args = { -# "vector_store_id": "book_2", -# "custom_llm_provider": "milvus", -# "api_base": "https://in03-18505f064ffbc6f.serverless.aws-eu-central-1.cloud.zilliz.com", -# "litellm_embedding_model": "text-embedding-3-large", -# "litellm_embedding_config": { -# "api_key": os.getenv("OPENAI_API_KEY"), -# }, -# "default_output_fields": [ -# "book_intro_text" -# ], # field containing the text to return in the response -# "default_anns_field": "book_intro_vector", -# } -# default_query = base_request_args.pop("query", "Basic ping") -# print(f"base_request_args: {base_request_args}") -# try: -# if sync_mode: -# response = vector_store_search(query=default_query, **base_request_args) -# else: -# response = await vector_store_asearch( -# query=default_query, **base_request_args -# ) -# except litellm.InternalServerError: -# pytest.skip("Skipping test due to litellm.InternalServerError") - -# print("litellm response=", json.dumps(response, indent=4, default=str)) -# assert len(response["data"]) > 0 # type: ignore - - -if __name__ == "__main__": - # Run tests - import asyncio - - test = TestMilvusVectorStore() - - print("Running async mock test...") - asyncio.run(test.test_basic_search_with_mock_async()) - - print("\nRunning sync mock test...") - test.test_basic_search_with_mock_sync() - - print("\n✅ All mock tests passed!") - - -class RecordingEmbeddingExecutor: - def __init__(self, response): - self.response = response - self.calls = [] - - def embed(self, model, query, configuration): - self.calls.append((model, query, dict(configuration))) - return self.response - - async def aembed(self, model, query, configuration): - self.calls.append((model, query, dict(configuration))) - return self.response - - -ALIAS_QUERY_VECTOR = [0.5, -0.25, 0.125] -ALIAS_EMBEDDING_RESPONSE = EmbeddingResponse( - data=[{"embedding": ALIAS_QUERY_VECTOR, "index": 0, "object": "embedding"}] -) -OPENAI_EMBEDDINGS_URL = "https://api.openai.com/v1/embeddings" -MILVUS_SEARCH_URL = "https://milvus.example/v2/vectordb/entities/search" -ALIAS_SEARCH_KWARGS = { - "query": "what is machine learning?", - "vector_store_id": "book_2", - "custom_llm_provider": "milvus", - "api_base": "https://milvus.example", - "api_key": "mock_milvus_api_key", - "litellm_embedding_model": "multilingual-e5-large", - "milvus_text_field": "book_intro_text", -} - - -def _alias_router(): - return Router( - model_list=[ - { - "model_name": "multilingual-e5-large", - "litellm_params": { - "model": "openai/text-embedding-3-small", - "api_key": "deployment-key", - }, - } - ] - ) - - -def _mock_embedding_route(respx_mock: respx.MockRouter) -> respx.Route: - return respx_mock.post(OPENAI_EMBEDDINGS_URL).mock( - return_value=httpx.Response( - 200, - json={ - "object": "list", - "data": [{"object": "embedding", "index": 0, "embedding": ALIAS_QUERY_VECTOR}], - "model": "text-embedding-3-small", - "usage": {"prompt_tokens": 2, "total_tokens": 2}, - }, - ) - ) - - -def _mock_search_route(respx_mock: respx.MockRouter) -> respx.Route: - return respx_mock.post(MILVUS_SEARCH_URL).mock(return_value=httpx.Response(200, json=MOCK_MILVUS_SEARCH_RESPONSE)) - - -def _assert_alias_resolved(embedding_route: respx.Route, search_route: respx.Route, response): - embedding_request = embedding_route.calls.last.request - assert embedding_request.headers["authorization"] == "Bearer deployment-key" - embedding_body = json.loads(embedding_request.read()) - assert embedding_body["model"] == "text-embedding-3-small" - assert embedding_body["input"] == ["what is machine learning?"] - search_request = search_route.calls.last.request - assert search_request.headers["authorization"] == "Bearer mock_milvus_api_key" - assert json.loads(search_request.read())["data"] == [ALIAS_QUERY_VECTOR] - assert len(response["data"]) == len(MOCK_MILVUS_SEARCH_RESPONSE["data"]) - assert response["data"][0]["content"][0]["text"] == MOCK_MILVUS_SEARCH_RESPONSE["data"][0]["book_intro_text"] - - -def test_router_search_resolves_bare_embedding_alias_sync( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch -): - monkeypatch.delenv("OPENAI_API_KEY", raising=False) - embedding_route = _mock_embedding_route(respx_mock) - search_route = _mock_search_route(respx_mock) - - response = _alias_router().vector_store_search(**ALIAS_SEARCH_KWARGS) - - _assert_alias_resolved(embedding_route, search_route, response) - - -@pytest.mark.asyncio -async def test_router_search_resolves_bare_embedding_alias_async( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch -): - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.delenv("OPENAI_API_KEY", raising=False) - embedding_route = _mock_embedding_route(respx_mock) - search_route = _mock_search_route(respx_mock) - - response = await _alias_router().avector_store_search(**ALIAS_SEARCH_KWARGS) - - _assert_alias_resolved(embedding_route, search_route, response) - - -def test_sdk_search_with_router_kwarg_resolves_bare_embedding_alias_sync( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch -): - monkeypatch.delenv("OPENAI_API_KEY", raising=False) - embedding_route = _mock_embedding_route(respx_mock) - search_route = _mock_search_route(respx_mock) - - response = litellm.vector_stores.search(router=_alias_router(), **ALIAS_SEARCH_KWARGS) - - _assert_alias_resolved(embedding_route, search_route, response) - - -@pytest.mark.asyncio -async def test_sdk_search_with_router_kwarg_resolves_bare_embedding_alias_async( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch -): - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.delenv("OPENAI_API_KEY", raising=False) - embedding_route = _mock_embedding_route(respx_mock) - search_route = _mock_search_route(respx_mock) - - response = await litellm.vector_stores.asearch(router=_alias_router(), **ALIAS_SEARCH_KWARGS) - - _assert_alias_resolved(embedding_route, search_route, response) - - -def _team_alias_router(): - return Router( - model_list=[ - { - "model_name": "team-a-embedder", - "litellm_params": { - "model": "openai/text-embedding-3-small", - "api_key": "deployment-key", - }, - "model_info": {"team_id": "team-a", "team_public_model_name": "multilingual-e5-large"}, - } - ] - ) - - -@pytest.mark.asyncio -async def test_sdk_search_with_router_kwarg_resolves_team_alias_from_request_metadata( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch -): - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.delenv("OPENAI_API_KEY", raising=False) - embedding_route = _mock_embedding_route(respx_mock) - search_route = _mock_search_route(respx_mock) - - response = await litellm.vector_stores.asearch( - router=_team_alias_router(), metadata={"user_api_key_team_id": "team-a"}, **ALIAS_SEARCH_KWARGS - ) - - _assert_alias_resolved(embedding_route, search_route, response) - - -@pytest.mark.asyncio -async def test_sdk_search_with_router_kwarg_rejects_team_alias_without_team_metadata( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch -): - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.delenv("OPENAI_API_KEY", raising=False) - embedding_route = _mock_embedding_route(respx_mock) - _mock_search_route(respx_mock) - - with pytest.raises(litellm.APIConnectionError): - await litellm.vector_stores.asearch(router=_team_alias_router(), **ALIAS_SEARCH_KWARGS) - - assert embedding_route.call_count == 0 - - -@pytest.mark.asyncio -async def test_transform_uses_injected_executor_without_embedding_config(respx_mock: respx.MockRouter): - executor = RecordingEmbeddingExecutor(ALIAS_EMBEDDING_RESPONSE) - config = MilvusVectorStoreConfig() - logging_obj = MagicMock() - logging_obj.model_call_details = {} - transform_kwargs = { - "vector_store_id": "book_2", - "query": ["what is", "milvus?"], - "vector_store_search_optional_params": {"limit": 3}, - "api_base": "https://milvus.example", - "litellm_logging_obj": logging_obj, - "litellm_params": {"litellm_embedding_model": "multilingual-e5-large", "milvus_db_name": "docs"}, - "embedding_executor": executor, - } - - url, sync_body = config.transform_search_vector_store_request(**transform_kwargs) - _, async_body = await config.atransform_search_vector_store_request(**transform_kwargs) - - assert respx_mock.calls.call_count == 0 - assert executor.calls == [("multilingual-e5-large", "what is milvus?", {})] * 2 - assert url == MILVUS_SEARCH_URL - assert sync_body == async_body - assert sync_body == { - "collectionName": "book_2", - "data": [ALIAS_QUERY_VECTOR], - "annsField": "book_intro_vector", - "limit": 3, - "dbName": "docs", - } - assert logging_obj.model_call_details["input"] == "what is milvus?" - assert logging_obj.model_call_details["embedding_model"] == "multilingual-e5-large" - - -def test_transform_falls_back_to_sdk_embedding_without_executor_or_config( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch -): - monkeypatch.setenv("OPENAI_API_KEY", "env-key") - embedding_route = _mock_embedding_route(respx_mock) - logging_obj = MagicMock() - logging_obj.model_call_details = {} - - _, body = MilvusVectorStoreConfig().transform_search_vector_store_request( - vector_store_id="book_2", - query="q", - vector_store_search_optional_params={}, - api_base="https://milvus.example", - litellm_logging_obj=logging_obj, - litellm_params={"litellm_embedding_model": "openai/text-embedding-3-small"}, - ) - - embedding_request = embedding_route.calls.last.request - assert embedding_request.headers["authorization"] == "Bearer env-key" - assert json.loads(embedding_request.read())["input"] == ["q"] - assert body["data"] == [ALIAS_QUERY_VECTOR] - - -def test_transform_requires_embedding_model(): - with pytest.raises(ValueError, match="litellm_embedding_model is required"): - MilvusVectorStoreConfig().transform_search_vector_store_request( - vector_store_id="book_2", - query="q", - vector_store_search_optional_params={}, - api_base="https://milvus.example", - litellm_logging_obj=MagicMock(), - litellm_params={"litellm_embedding_config": {"api_key": "store-key"}}, - embedding_executor=RecordingEmbeddingExecutor(ALIAS_EMBEDDING_RESPONSE), - ) diff --git a/tests/vector_store_tests/test_openai_vector_store.py b/tests/vector_store_tests/test_openai_vector_store.py deleted file mode 100644 index 6327b45eaa1..00000000000 --- a/tests/vector_store_tests/test_openai_vector_store.py +++ /dev/null @@ -1,20 +0,0 @@ -from base_vector_store_test import BaseVectorStoreTest - - -class TestOpenAIVectorStore(BaseVectorStoreTest): - def get_base_request_args(self) -> dict: - """ - This is a real vector store on OpenAI - """ - return { - "vector_store_id": "vs_685b14b1a1b88191bc27e04f1917fddd", - "custom_llm_provider": "openai", - } - - def get_base_create_vector_store_args(self) -> dict: - """ - This is a real vector store on OpenAI - """ - return { - "custom_llm_provider": "openai", - } diff --git a/tests/vector_store_tests/test_ragflow_vector_store.py b/tests/vector_store_tests/test_ragflow_vector_store.py deleted file mode 100644 index c0511ebacc6..00000000000 --- a/tests/vector_store_tests/test_ragflow_vector_store.py +++ /dev/null @@ -1,377 +0,0 @@ -""" -Test RAGFlow Vector Store helper functions and transformation. -""" - -import os -import json -import pytest -from unittest.mock import Mock, patch, MagicMock -import httpx - -import litellm - -from tests.vector_store_tests.base_vector_store_test import BaseVectorStoreTest -from litellm.llms.ragflow.vector_stores.transformation import RAGFlowVectorStoreConfig -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.types.vector_stores import VectorStoreCreateOptionalRequestParams -from litellm.llms.base_llm.chat.transformation import BaseLLMException - - -class TestRAGFlowVectorStore(BaseVectorStoreTest): - """ - Test the RAGFlow vector store transformation functionality. - """ - - def get_base_create_vector_store_args(self) -> dict: - """Must return the base create vector store args""" - return { - "custom_llm_provider": "ragflow", - "api_key": os.getenv("RAGFLOW_API_KEY", "test-api-key"), - "api_base": os.getenv("RAGFLOW_API_BASE", "http://localhost:9380"), - } - - def get_base_request_args(self): - # RAGFlow doesn't support search, so we'll skip search tests - return { - "vector_store_id": "test-dataset-id", - "custom_llm_provider": "ragflow", - "query": "test query", - } - - def test_get_auth_credentials(self): - """Test that auth credentials are correctly extracted.""" - config = RAGFlowVectorStoreConfig() - - # Test with api_key in params - litellm_params = {"api_key": "test-api-key-123"} - credentials = config.get_auth_credentials(litellm_params) - assert "headers" in credentials - assert credentials["headers"]["Authorization"] == "Bearer test-api-key-123" - - # Test with missing api_key (should raise ValueError) - with pytest.raises(ValueError, match="api_key is required"): - config.get_auth_credentials({}) - - def test_get_complete_url(self): - """Test that complete URL is correctly constructed.""" - config = RAGFlowVectorStoreConfig() - - # Test with api_base in params - litellm_params = {"api_base": "http://custom-host:9999"} - url = config.get_complete_url(api_base=None, litellm_params=litellm_params) - assert url == "http://custom-host:9999/api/v1/datasets" - - # Test with api_base parameter - url = config.get_complete_url( - api_base="http://test-host:8888", litellm_params={} - ) - assert url == "http://test-host:8888/api/v1/datasets" - - # Test with default (no api_base provided) - with patch.dict(os.environ, {}, clear=True): - url = config.get_complete_url(api_base=None, litellm_params={}) - assert url == "http://localhost:9380/api/v1/datasets" - - # Test with trailing slash removal - url = config.get_complete_url( - api_base="http://test-host:8888/", litellm_params={} - ) - assert url == "http://test-host:8888/api/v1/datasets" - - def test_validate_environment(self): - """Test environment validation and header setting.""" - config = RAGFlowVectorStoreConfig() - from litellm.types.router import GenericLiteLLMParams - - # Test with api_key in litellm_params - litellm_params = GenericLiteLLMParams(api_key="test-key") - headers = config.validate_environment({}, litellm_params) - assert headers["Authorization"] == "Bearer test-key" - assert headers["Content-Type"] == "application/json" - - # Test with missing api_key - with pytest.raises(ValueError, match="RAGFLOW_API_KEY"): - config.validate_environment({}, GenericLiteLLMParams()) - - def test_get_vector_store_endpoints_by_type(self): - """Test that endpoints are correctly configured (empty for management only).""" - config = RAGFlowVectorStoreConfig() - endpoints = config.get_vector_store_endpoints_by_type() - assert endpoints["read"] == [] - assert endpoints["write"] == [] - - def test_transform_create_vector_store_request_basic(self): - """Test basic dataset creation request transformation.""" - config = RAGFlowVectorStoreConfig() - - params: VectorStoreCreateOptionalRequestParams = {"name": "test-dataset"} - - url, body = config.transform_create_vector_store_request( - params, "http://localhost:9380/api/v1/datasets" - ) - - assert url == "http://localhost:9380/api/v1/datasets" - assert body["name"] == "test-dataset" - assert body["chunk_method"] == "naive" # Default chunk method - - def test_transform_create_vector_store_request_with_metadata(self): - """Test dataset creation with RAGFlow-specific metadata.""" - config = RAGFlowVectorStoreConfig() - - params: VectorStoreCreateOptionalRequestParams = { - "name": "test-dataset-advanced", - "metadata": { - "description": "Test dataset", - "embedding_model": "BAAI/bge-large-zh-v1.5@BAAI", - "permission": "me", - "chunk_method": "naive", - "parser_config": {"chunk_token_num": 512, "delimiter": "\n"}, - }, - } - - url, body = config.transform_create_vector_store_request( - params, "http://localhost:9380/api/v1/datasets" - ) - - assert body["name"] == "test-dataset-advanced" - assert body["description"] == "Test dataset" - assert body["embedding_model"] == "BAAI/bge-large-zh-v1.5@BAAI" - assert body["permission"] == "me" - assert body["chunk_method"] == "naive" - assert "parser_config" in body - assert body["parser_config"]["chunk_token_num"] == 512 - - def test_transform_create_vector_store_request_missing_name(self): - """Test that missing name raises ValueError.""" - config = RAGFlowVectorStoreConfig() - - params: VectorStoreCreateOptionalRequestParams = {} - - with pytest.raises(ValueError, match="name is required"): - config.transform_create_vector_store_request( - params, "http://localhost:9380/api/v1/datasets" - ) - - def test_transform_create_vector_store_request_mutually_exclusive(self): - """Test that chunk_method and pipeline_id are mutually exclusive.""" - config = RAGFlowVectorStoreConfig() - - params: VectorStoreCreateOptionalRequestParams = { - "name": "test-dataset", - "metadata": { - "chunk_method": "naive", - "pipeline_id": "d0bebe30ae2211f0970942010a8e0005", - }, - } - - with pytest.raises(ValueError, match="mutually exclusive"): - config.transform_create_vector_store_request( - params, "http://localhost:9380/api/v1/datasets" - ) - - def test_transform_create_vector_store_request_with_pipeline(self): - """Test dataset creation with ingestion pipeline.""" - config = RAGFlowVectorStoreConfig() - - params: VectorStoreCreateOptionalRequestParams = { - "name": "test-pipeline-dataset", - "metadata": { - "parse_type": 2, - "pipeline_id": "d0bebe30ae2211f0970942010a8e0005", - }, - } - - url, body = config.transform_create_vector_store_request( - params, "http://localhost:9380/api/v1/datasets" - ) - - assert body["name"] == "test-pipeline-dataset" - assert body["parse_type"] == 2 - assert body["pipeline_id"] == "d0bebe30ae2211f0970942010a8e0005" - assert "chunk_method" not in body - - def test_transform_create_vector_store_response_success(self): - """Test successful response transformation.""" - config = RAGFlowVectorStoreConfig() - - # Mock RAGFlow response - mock_response = Mock(spec=httpx.Response) - mock_response.status_code = 200 - mock_response.headers = {} - mock_response.json.return_value = { - "code": 0, - "data": { - "id": "3b4de7d4241d11f0a6a79f24fc270c7f", - "name": "test-dataset", - "create_time": 1745836841611, - "chunk_method": "naive", - "embedding_model": "BAAI/bge-large-zh-v1.5@BAAI", - }, - } - - response = config.transform_create_vector_store_response(mock_response) - - assert response["id"] == "3b4de7d4241d11f0a6a79f24fc270c7f" - assert response["name"] == "test-dataset" - assert response["object"] == "vector_store" - assert response["status"] == "completed" - assert response["created_at"] == 1745836841 # Converted from milliseconds - assert response["bytes"] == 0 - assert "file_counts" in response - - def test_transform_create_vector_store_response_error(self): - """Test error response transformation.""" - config = RAGFlowVectorStoreConfig() - - # Mock RAGFlow error response - mock_response = Mock(spec=httpx.Response) - mock_response.status_code = 400 - mock_response.headers = {} - mock_response.json.return_value = { - "code": 101, - "message": "Dataset name 'test-dataset' already exists", - } - - with pytest.raises(BaseLLMException): - config.transform_create_vector_store_response(mock_response) - - def test_transform_create_vector_store_response_missing_id(self): - """Test response with missing dataset ID.""" - config = RAGFlowVectorStoreConfig() - - mock_response = Mock(spec=httpx.Response) - mock_response.status_code = 200 - mock_response.headers = {} - mock_response.json.return_value = { - "code": 0, - "data": { - "name": "test-dataset" - # Missing "id" - }, - } - - with pytest.raises(ValueError, match="missing dataset id"): - config.transform_create_vector_store_response(mock_response) - - def test_transform_search_vector_store_request_not_implemented(self): - """Test that search operations raise NotImplementedError.""" - config = RAGFlowVectorStoreConfig() - logging_obj = MagicMock(spec=LiteLLMLoggingObj) - - with pytest.raises(NotImplementedError, match="management only"): - config.transform_search_vector_store_request( - vector_store_id="test-id", - query="test query", - vector_store_search_optional_params={}, - api_base="http://localhost:9380", - litellm_logging_obj=logging_obj, - litellm_params={}, - extra_body=None, - ) - - def test_transform_search_vector_store_response_not_implemented(self): - """Test that search response transformation raises NotImplementedError.""" - config = RAGFlowVectorStoreConfig() - logging_obj = MagicMock(spec=LiteLLMLoggingObj) - mock_response = Mock(spec=httpx.Response) - - with pytest.raises(NotImplementedError, match="management only"): - config.transform_search_vector_store_response(mock_response, logging_obj) - - def _validate_vector_store_create_response(self, response): - """Override to handle RAGFlow-specific response format.""" - # RAGFlow IDs are hex strings (not OpenAI-style vs_* format) - # So we override the base validation to not check for vs_ prefix - assert isinstance( - response, dict - ), f"Response should be a dict, got {type(response)}" - assert "id" in response, "Missing required field 'id' in create response" - assert ( - "object" in response - ), "Missing required field 'object' in create response" - assert ( - "created_at" in response - ), "Missing required field 'created_at' in create response" - - assert ( - response["object"] == "vector_store" - ), f"Expected object to be 'vector_store', got '{response['object']}'" - - assert isinstance( - response["id"], str - ), f"id should be a string, got {type(response['id'])}" - assert len(response["id"]) > 0, "id should not be empty" - # RAGFlow IDs are hex strings, not OpenAI-style vs_* format - - assert isinstance( - response["created_at"], int - ), f"created_at should be an integer, got {type(response['created_at'])}" - assert response["created_at"] > 0, "created_at should be a positive timestamp" - - print( - f"✅ RAGFlow create response validation passed: Dataset '{response['id']}' created successfully" - ) - - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - async def test_basic_create_vector_store(self, sync_mode): - """Override to handle RAGFlow-specific connection errors.""" - litellm.turn_on_debug() - litellm.set_verbose = True - base_request_args = self.get_base_create_vector_store_args() - - # Skip if no API key is set - if not os.getenv("RAGFLOW_API_KEY") and not base_request_args.get("api_key"): - pytest.skip("RAGFLOW_API_KEY not set, skipping integration test") - - # Extract custom_llm_provider from base args if present - create_args = base_request_args - try: - if sync_mode: - response = litellm.vector_stores.create( - name=f"test-ragflow-{int(__import__('time').time())}", **create_args - ) - else: - response = await litellm.vector_stores.acreate( - name=f"test-ragflow-{int(__import__('time').time())}", **create_args - ) - except litellm.InternalServerError: - pytest.skip("Skipping test due to litellm.InternalServerError") - except Exception as e: - error_str = str(e).lower() - error_type = type(e).__name__ - - # Check if it's a connection error - if ( - isinstance(e, (ConnectionError, OSError)) - or "connection" in error_str - or "connect" in error_str - or "APIConnectionError" in error_type - ): - pytest.skip( - f"Skipping test due to connection error (RAGFlow instance may not be running): {e}" - ) - - # If this is an authentication or permission error, skip the test - if ( - "authentication" in error_str - or "permission" in error_str - or "unauthorized" in error_str - ): - pytest.skip( - f"Skipping test due to authentication/permission error: {e}" - ) - - # Re-raise if it's not a handled error - raise - - print("litellm create response=", json.dumps(response, indent=4, default=str)) - - # Validate response structure - self._validate_vector_store_create_response(response) - - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - async def test_basic_search_vector_store(self, sync_mode): - """Override search test - RAGFlow doesn't support search.""" - pytest.skip("RAGFlow vector stores support dataset management only, not search") diff --git a/tests/vector_store_tests/test_s3_vectors_vector_store.py b/tests/vector_store_tests/test_s3_vectors_vector_store.py deleted file mode 100644 index e99af039d68..00000000000 --- a/tests/vector_store_tests/test_s3_vectors_vector_store.py +++ /dev/null @@ -1,44 +0,0 @@ -from base_vector_store_test import BaseVectorStoreTest -import os -import pytest - - -class TestS3VectorsVectorStore(BaseVectorStoreTest): - @pytest.fixture(autouse=True) - def check_env_vars(self): - """Check if required environment variables are set""" - required_vars = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] - missing_vars = [var for var in required_vars if not os.getenv(var)] - if missing_vars: - pytest.skip( - f"Missing required environment variables: {', '.join(missing_vars)}" - ) - - def get_base_request_args(self) -> dict: - """ - Must return the base request args for searching. - For S3 Vectors, vector_store_id should be in format: bucket_name:index_name - """ - return { - "custom_llm_provider": "s3_vectors", - "vector_store_id": os.getenv( - "S3_VECTORS_VECTOR_STORE_ID", "test-litellm-vectors:test-index" - ), - "query": "What is machine learning?", - "aws_region_name": os.getenv("AWS_REGION_NAME", "us-west-2"), - "aws_access_key_id": os.getenv("AWS_ACCESS_KEY_ID"), - "aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY"), - } - - def get_base_create_vector_store_args(self) -> dict: - """ - Vector store creation is not yet implemented for S3 Vectors. - This test will be skipped. - """ - return {} - - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - async def test_basic_create_vector_store(self, sync_mode): - """S3 Vectors doesn't support vector store creation via this API yet""" - pytest.skip("Vector store creation not yet implemented for S3 Vectors") diff --git a/tests/vector_store_tests/test_vertex_ai_search_api_vector_store.py b/tests/vector_store_tests/test_vertex_ai_search_api_vector_store.py deleted file mode 100644 index 7cace338616..00000000000 --- a/tests/vector_store_tests/test_vertex_ai_search_api_vector_store.py +++ /dev/null @@ -1,194 +0,0 @@ -""" -Test for Vertex AI Search API Vector Store with mocked responses -""" - -import json -import pytest -from unittest.mock import AsyncMock, MagicMock, patch -import litellm - - -# Mock response from actual Vertex AI Search API -MOCK_VERTEX_SEARCH_RESPONSE = { - "results": [ - { - "id": "0", - "document": { - "name": "projects/648660250433/locations/global/collections/default_collection/dataStores/litellm-docs_1761094140318/branches/0/documents/0", - "id": "0", - "derivedStructData": { - "htmlTitle": "LiteLLM - Getting Started | liteLLM", - "snippets": [ - { - "htmlSnippet": "https://github.com/BerriAI/litellm.", - "snippet": "https://github.com/BerriAI/litellm.", - } - ], - "title": "LiteLLM - Getting Started | liteLLM", - "link": "https://docs.litellm.ai/docs/", - "displayLink": "docs.litellm.ai", - }, - }, - }, - { - "id": "1", - "document": { - "name": "projects/648660250433/locations/global/collections/default_collection/dataStores/litellm-docs_1761094140318/branches/0/documents/1", - "id": "1", - "derivedStructData": { - "title": "Using Vector Stores (Knowledge Bases) | liteLLM", - "link": "https://docs.litellm.ai/docs/completion/knowledgebase", - "snippets": [ - { - "snippet": "LiteLLM integrates with vector stores, allowing your models to access your organization's data for more accurate and contextually relevant responses." - } - ], - }, - }, - }, - ], - "totalSize": 299, - "attributionToken": "mock_token", - "summary": {}, -} - - -class TestVertexAISearchAPIVectorStore: - """Test Vertex AI Search API Vector Store with mocked responses""" - - @pytest.mark.asyncio - async def test_basic_search_with_mock(self): - """Test basic vector search with mocked backend response""" - - # Mock the HTTP response - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = MOCK_VERTEX_SEARCH_RESPONSE - mock_response.text = json.dumps(MOCK_VERTEX_SEARCH_RESPONSE) - - # Mock the access token method to avoid real authentication - with patch( - "litellm.llms.vertex_ai.vector_stores.search_api.transformation.VertexSearchAPIVectorStoreConfig._ensure_access_token" - ) as mock_auth: - mock_auth.return_value = ("mock_token", "test-vector-store-db") - - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - new_callable=AsyncMock, - ) as mock_post: - mock_post.return_value = mock_response - - # Make the search request - response = await litellm.vector_stores.asearch( - query="what is LiteLLM?", - vector_store_id="test-litellm-app_1761094730750", - custom_llm_provider="vertex_ai/search_api", - vertex_project="test-vector-store-db", - vertex_location="us-central1", - ) - - print("Response:", json.dumps(response, indent=2, default=str)) - - # Validate the response structure (LiteLLM standard format) - assert response is not None - assert response["object"] == "vector_store.search_results.page" - assert "data" in response - assert len(response["data"]) > 0 - assert "search_query" in response - - # Validate first result - first_result = response["data"][0] - assert "score" in first_result - assert "content" in first_result - assert "file_id" in first_result - assert "filename" in first_result - assert "attributes" in first_result - - # Validate content structure - assert len(first_result["content"]) > 0 - assert first_result["content"][0]["type"] == "text" - assert "text" in first_result["content"][0] - - # Verify the API was called - mock_post.assert_called_once() - - # Verify the URL format - call_args = mock_post.call_args - url = call_args[1]["url"] if "url" in call_args[1] else call_args[0][0] - assert "discoveryengine.googleapis.com" in url - assert "test-vector-store-db" in url - assert "test-litellm-app_1761094730750" in url - - def test_basic_search_sync_with_mock(self): - """Test basic vector search (sync) with mocked backend response""" - - # Mock the HTTP response - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = MOCK_VERTEX_SEARCH_RESPONSE - mock_response.text = json.dumps(MOCK_VERTEX_SEARCH_RESPONSE) - - # Mock the access token method to avoid real authentication - with patch( - "litellm.llms.vertex_ai.vector_stores.search_api.transformation.VertexSearchAPIVectorStoreConfig._ensure_access_token" - ) as mock_auth: - mock_auth.return_value = ("mock_token", "test-vector-store-db") - - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post" - ) as mock_post: - mock_post.return_value = mock_response - - # Make the search request - response = litellm.vector_stores.search( - query="what is LiteLLM?", - vector_store_id="test-litellm-app_1761094730750", - custom_llm_provider="vertex_ai/search_api", - vertex_project="test-vector-store-db", - vertex_location="us-central1", - ) - - print("Response:", json.dumps(response, indent=2, default=str)) - - # Validate the response structure (LiteLLM standard format) - assert response is not None - assert response["object"] == "vector_store.search_results.page" - assert "data" in response - assert len(response["data"]) > 0 - assert "search_query" in response - - # Validate first result structure - first_result = response["data"][0] - assert "score" in first_result - assert "content" in first_result - assert "file_id" in first_result - assert "filename" in first_result - assert "attributes" in first_result - - # Validate content structure - assert len(first_result["content"]) > 0 - assert first_result["content"][0]["type"] == "text" - assert "text" in first_result["content"][0] - - # Validate attributes - assert "document_id" in first_result["attributes"] - assert "link" in first_result["attributes"] - assert "title" in first_result["attributes"] - - # Verify the API was called - mock_post.assert_called_once() - - -if __name__ == "__main__": - # Run tests - import asyncio - - test = TestVertexAISearchAPIVectorStore() - - print("Running async test...") - asyncio.run(test.test_basic_search_with_mock()) - - print("\nRunning sync test...") - test.test_basic_search_sync_with_mock() - - print("\n✅ All tests passed!") diff --git a/tests/vector_store_tests/test_vertex_ai_vector_store.py b/tests/vector_store_tests/test_vertex_ai_vector_store.py deleted file mode 100644 index 9dcbe4956a2..00000000000 --- a/tests/vector_store_tests/test_vertex_ai_vector_store.py +++ /dev/null @@ -1,27 +0,0 @@ -import os -import pytest -from unittest.mock import Mock, patch - -from litellm.llms.vertex_ai.vector_stores.transformation import VertexVectorStoreConfig -from litellm.types.vector_stores import ( - VectorStoreCreateResponse, - VectorStoreSearchResponse, - VectorStoreSearchResult, - VectorStoreResultContent, -) -from tests.vector_store_tests.base_vector_store_test import BaseVectorStoreTest - - -class TestVertexAIVectorStore(BaseVectorStoreTest): - def get_base_create_vector_store_args(self) -> dict: - """Must return the base create vector store args""" - return {} - - def get_base_request_args(self): - return { - "vector_store_id": "6917529027641081856", - "custom_llm_provider": "vertex_ai", - "vertex_project": "reliablekeys", - "vertex_location": "us-central1", - "query": "what happens after we add a model", - } diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 96517aabb46..60c85e2a926 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -13669,10 +13669,10 @@ export interface paths { * Get Public Whats New * @description Return the launches the dashboard Home page shows under What's new. * - * Resolved once per process: fetched from ``litellm.whats_new_url`` (override with ``LITELLM_WHATS_NEW_URL``) - * on the first request, falling back to the list bundled with the package on any failure, so an airgapped - * proxy serves the bundled list without retrying. Set ``LITELLM_LOCAL_WHATS_NEW=True`` to skip the fetch. - * A restart picks up a newly published list. + * Serves the list bundled with this release. Set ``LITELLM_WHATS_NEW_URL`` to fetch a remote list instead, + * resolved once per process on the first request and falling back to the bundled list on any failure, so an + * airgapped proxy serves the bundled list without retrying. ``LITELLM_LOCAL_WHATS_NEW=True`` forces the + * bundled list even when a URL is set. A restart picks up a newly published list. */ get: operations["get_public_whats_new_public_whats_new_get"]; put?: never;