diff --git a/.circleci/config.yml b/.circleci/config.yml index f2d51ba7a0f..2653135da41 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3292,35 +3292,6 @@ jobs: - store_artifacts: path: test-results - unit: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - parallelism: 4 - steps: - - setup_litellm_test_deps - - run: - name: Generate Prisma client - command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - - run: - name: Run unit tests - command: | - mkdir -p test-results/unit - shard="$(find tests/unit -name 'test_*.py' | sort | circleci tests split --split-by=timings --timings-type=filename)" - if [ -z "${shard}" ]; then echo "shard ${CIRCLE_NODE_INDEX} received no tests/unit files; nothing to run"; exit 0; fi - mapfile -t files < <(printf '%s\n' "${shard}") - set +e - LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short -o junit_family=xunit1 --junitxml=test-results/unit/junit.xml - status=$? - set -e - if [ "$status" -eq 5 ]; then echo "pytest collected no tests from tests/unit; passing"; exit 0; fi - exit "$status" - - store_test_results: - path: test-results - - store_artifacts: - path: test-results - postgres_suite: parameters: test_path: @@ -3594,7 +3565,6 @@ workflows: jobs: - using_litellm_on_windows - windows_release_wheel - - unit - provider_replay_harness - base_sdk_install - local_testing_part1 diff --git a/.circleci/scripts/unit_selection.sh b/.circleci/scripts/unit_selection.sh deleted file mode 100755 index 02df32d5eab..00000000000 --- a/.circleci/scripts/unit_selection.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -flag="${1:?usage: unit_selection.sh }" - -legacy_flags=( - caching-local - core-utils - enterprise-package - enterprise-routing - integrations - llm-other-providers - llm-vertex-ai - mcp-integration - misc - proxy-db-auth-checks - proxy-db-budgets - proxy-db-custom-logging - proxy-db-db-and-spend - proxy-db-endpoints-and-responses - proxy-db-guardrails-hooks - proxy-db-jwt-and-keys - proxy-db-key-generation - proxy-db-logging-misc - proxy-db-proxy-runtime - proxy-db-proxy-server-core - proxy-db-proxy-utils - proxy-extras - proxy-infra - responses-caching-types -) - -legacy_paths() { - case "$1" in - caching-local) echo tests/unit/caching ;; - core-utils) echo tests/unit/litellm_core_utils ;; - enterprise-package) - echo tests/unit/enterprise/integrations - echo tests/unit/enterprise/proxy/auth - echo tests/unit/enterprise/proxy/guardrails - echo tests/unit/enterprise/proxy/hooks - echo tests/unit/enterprise/proxy/management_endpoints - echo tests/unit/enterprise/proxy/test_audit_logging_endpoints.py - echo tests/unit/enterprise/proxy/test_liteadmin.py - echo tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py ;; - enterprise-routing) - echo tests/unit/google_genai - echo tests/unit/router_strategy - echo tests/unit/router_utils - echo tests/unit/proxy/common_utils/test_cache_aware_routing.py - echo tests/unit/enterprise/enterprise_callbacks/send_emails - echo tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py - echo tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py - echo tests/unit/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py - echo tests/unit/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py - echo tests/unit/enterprise/proxy/test_batch_update_db_managed_output_file_id.py - echo tests/unit/enterprise/proxy/test_deleted_file_returns_403_not_404.py - echo tests/unit/enterprise/proxy/test_enterprise_routes.py - echo tests/unit/enterprise/proxy/test_file_deletion_blocking.py - echo tests/unit/enterprise/proxy/test_managed_files_access_check.py - echo tests/unit/enterprise/proxy/test_managed_files_hook.py ;; - integrations) echo tests/unit/integrations ;; - llm-other-providers) find tests/unit/llms -name 'test_*.py' -not -path 'tests/unit/llms/vertex_ai/*' ;; - llm-vertex-ai) echo tests/unit/llms/vertex_ai ;; - mcp-integration) - echo tests/unit/experimental_mcp_client - echo tests/unit/proxy/_experimental/mcp_server - echo tests/unit/responses/mcp - echo tests/mcp_tests/test_proxy_mcp_e2e.py ;; - misc) - find tests/unit -maxdepth 1 -name 'test_*.py' - echo tests/unit/test_router - echo tests/unit/a2a_protocol - echo tests/unit/batches - echo tests/unit/chat_completions - echo tests/unit/completion_extras - echo tests/unit/containers - echo tests/unit/embeddings - echo tests/unit/endpoints - echo tests/unit/files - echo tests/unit/harness - echo tests/unit/images - echo tests/unit/interactions - echo tests/unit/messages - echo tests/unit/rag - echo tests/unit/rerank_api - echo tests/unit/rust_bridge - echo tests/unit/secret_managers - echo tests/unit/vector_stores - echo tests/unit/videos ;; - proxy-db-auth-checks) - echo tests/unit/proxy/auth/test_auth_checks.py - echo tests/unit/proxy/auth/test_user_api_key_auth.py - echo tests/unit/proxy/test_credential_slot_registry.py - echo tests/unit/proxy/test_deprecated_key_grace_period.py ;; - proxy-db-budgets) - echo tests/unit/proxy/auth/test_default_end_user_budget_simple.py - echo tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py - echo tests/unit/proxy/test_zero_cost_model_budget_bypass.py ;; - proxy-db-custom-logging) - echo tests/unit/proxy/test_custom_callback_input.py - echo tests/unit/proxy/test_custom_logger_s3_gcs.py ;; - proxy-db-db-and-spend) - echo tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py - echo tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py - echo tests/unit/proxy/db/test_update_daily_tag_spend.py - echo tests/unit/proxy/test_db_schema_changes.py - echo tests/unit/proxy/test_prisma_client_backoff_retry.py - echo tests/unit/proxy/test_update_spend.py - echo tests/unit/skills/test_skills_db.py ;; - proxy-db-endpoints-and-responses) - echo tests/unit/proxy/lens - echo tests/unit/proxy/auth/test_models_fallback_endpoint.py - echo tests/unit/proxy/common_utils/test_check_batch_cost.py - echo tests/unit/proxy/common_utils/test_check_responses_cost.py - echo tests/unit/proxy/common_utils/test_realtime_cache.py - echo tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py - echo tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py - echo tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py - echo tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py - echo tests/unit/proxy/response_polling - echo tests/unit/proxy/test_custom_tokenizer_bug.py - echo tests/unit/proxy/test_get_favicon.py - echo tests/unit/proxy/test_get_image.py - echo tests/unit/proxy/test_prompt_test_endpoint.py - echo tests/unit/proxy/test_reducto_ocr_route.py - echo tests/unit/proxy/test_response_polling_pre_call_checks.py - echo tests/unit/proxy/test_ui_path_detection.py ;; - proxy-db-guardrails-hooks) - echo tests/unit/proxy/hooks/test_banned_keyword_list.py - echo tests/unit/proxy/test_proxy_setting_guardrails.py - echo tests/unit/proxy/test_unit_test_proxy_hooks.py ;; - proxy-db-jwt-and-keys) - echo tests/unit/proxy/auth/test_jwt.py - echo tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py - echo tests/unit/proxy/test_proxy_custom_auth.py ;; - proxy-db-key-generation) echo tests/unit/proxy/management_endpoints/test_key_generate_prisma.py ;; - proxy-db-logging-misc) - echo tests/unit/proxy/management_helpers/test_audit_logs_proxy.py - echo tests/unit/proxy/spend_tracking/test_search_api_logging.py - echo tests/unit/proxy/test_proxy_reject_logging.py ;; - proxy-db-proxy-runtime) - echo tests/unit/proxy/auth/test_multipart_bypass_repro.py - echo tests/unit/proxy/auth/test_proxy_routes.py - echo tests/unit/proxy/middleware/test_request_size_limit_middleware.py - echo tests/unit/proxy/test_proxy_config_unit_test.py - echo tests/unit/proxy/test_proxy_token_counter.py - echo tests/unit/proxy/test_server_root_path.py ;; - proxy-db-proxy-server-core) - echo tests/unit/proxy/test__lazy_features.py - echo tests/unit/proxy/test_aproxy_startup.py - echo tests/unit/proxy/test_proxy_server.py ;; - proxy-db-proxy-utils) echo tests/unit/proxy/test_proxy_utils.py ;; - proxy-extras) echo tests/unit/litellm_proxy_extras ;; - proxy-infra) - echo tests/unit/gateway - echo tests/unit/proxy/management - echo tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py - echo tests/unit/proxy/roi_calculator ;; - responses-caching-types) - find tests/unit/responses -name 'test_*.py' -not -path 'tests/unit/responses/mcp/*' - echo tests/unit/types ;; - *) echo "unit_selection.sh: unknown flag $1" >&2; exit 1 ;; - esac -} - -expand() { - while read -r path; do - if [ -d "$path" ]; then - find "$path" -name 'test_*.py' - elif [ -f "$path" ]; then - echo "$path" - else - echo "unit_selection.sh: $path does not exist" >&2 - exit 1 - fi - done -} - -if [ "$flag" = unit ]; then - comm -23 \ - <(find tests/unit -name 'test_*.py' | sort) \ - <(for legacy in "${legacy_flags[@]}"; do legacy_paths "$legacy"; done | expand | sort) - exit 0 -fi - -legacy_paths "$flag" | expand | sort diff --git a/.circleci/tests.yml b/.circleci/tests.yml deleted file mode 100644 index a9cd21bad5e..00000000000 --- a/.circleci/tests.yml +++ /dev/null @@ -1,416 +0,0 @@ -version: 2.1 - -commands: - wait_for_service: - parameters: - url: - type: string - timeout: - type: string - default: "60" - steps: - - run: - name: "Wait for << parameters.url >>" - command: | - TIMEOUT=<< parameters.timeout >> - URL="<< parameters.url >>" - ELAPSED=0 - echo "Waiting up to ${TIMEOUT}s for ${URL} ..." - if echo "$URL" | grep -q '^tcp://'; then - HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1) - PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2) - while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do - sleep 2; ELAPSED=$((ELAPSED+2)) - if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi - done - else - while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do - sleep 2; ELAPSED=$((ELAPSED+2)) - if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi - done - fi - echo "Service ready after ${ELAPSED}s" - install_uv: - steps: - - run: - name: Install uv (pinned 0.10.9) - command: | - curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh - echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh - rm -f /tmp/uv-install.sh - echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" - export PATH="$HOME/.local/bin:$PATH" - install_rust: - steps: - - run: - name: Install Rust (rustup 1.28.2, toolchain 1.98.0) - command: | - case "$(uname -m)" in - x86_64) - RUSTUP_TRIPLE=x86_64-unknown-linux-gnu - RUSTUP_SHA256=20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c - ;; - aarch64) - RUSTUP_TRIPLE=aarch64-unknown-linux-gnu - RUSTUP_SHA256=e3853c5a252fca15252d07cb23a1bdd9377a8c6f3efa01531109281ae47f841c - ;; - *) - echo "install_rust: unsupported architecture $(uname -m)" >&2 - exit 1 - ;; - esac - curl -sSLf -o /tmp/rustup-init \ - "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init" - echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c - - chmod +x /tmp/rustup-init - /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0 - rm -f /tmp/rustup-init - echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV" - export PATH="$HOME/.cargo/bin:$PATH" - rustc --version - cargo --version - install_codecov_cli: - steps: - - run: - name: Install Codecov CLI (pinned v11.3.1) - when: always - command: | - curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov - curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM - [ "$(cat /tmp/codecov.SHA256SUM)" = "ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d codecov" ] - (cd /tmp && sha256sum -c codecov.SHA256SUM) - chmod +x /tmp/codecov - mkdir -p "$HOME/.local/bin" - mv /tmp/codecov "$HOME/.local/bin/codecov" - setup_litellm_enterprise_pip: - steps: - - run: - name: "Install local version of litellm-enterprise" - command: | - uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)" - setup_test_deps: - steps: - - install_uv - - install_rust - - restore_cache: - keys: - - v3-integration-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ~/.cache/uv - key: v3-integration-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Generate Prisma client - command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - skip_unless_relevant: - parameters: - category: - type: string - default: backend - base_ref: - type: string - default: "" - pull_request_url: - type: string - default: "" - steps: - - run: - name: "Skip job when no << parameters.category >>-relevant files changed" - command: | - export CIRCLE_PULL_REQUEST="${CIRCLE_PULL_REQUEST:-<< parameters.pull_request_url >>}" - export PATH_FILTER_BASE_BRANCH="<< parameters.base_ref >>" - [ -n "$PATH_FILTER_BASE_BRANCH" ] || unset PATH_FILTER_BASE_BRANCH - bash .circleci/scripts/path_filter.sh << parameters.category >> - start_postgres: - parameters: - db_name: - type: string - default: circle_test - image: - type: string - default: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26 - steps: - - run: - name: Start PostgreSQL - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=<< parameters.db_name >> \ - -p 5432:5432 \ - << parameters.image >> - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - start_redis: - steps: - - run: - name: Start Redis - command: | - docker run -d \ - --name redis-cache \ - -p 6379:6379 \ - redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6 - - wait_for_service: - url: tcp://localhost:6379 - timeout: "60" - -jobs: - unit: - parameters: - flag: - type: string - default: unit - shards: - type: integer - default: 6 - workers: - type: integer - default: 4 - dist: - type: string - default: loadscope - base_ref: - type: string - default: "" - pull_request_url: - type: string - default: "" - legacy_mcp_peer: - type: boolean - default: false - reruns: - type: integer - default: 0 - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - parallelism: << parameters.shards >> - environment: - COVERAGE_CORE: sysmon - LITELLM_LOCAL_MODEL_COST_MAP: "True" - steps: - - checkout - - skip_unless_relevant: - base_ref: << parameters.base_ref >> - pull_request_url: << parameters.pull_request_url >> - - setup_test_deps - - when: - condition: << parameters.legacy_mcp_peer >> - steps: - - run: - name: Install MCP SDK1 peer - command: | - uv venv --python 3.12 .venv-mcp-peer - uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1' - echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV" - - run: - name: "Run << parameters.flag >> shard" - no_output_timeout: 20m - command: | - mkdir -p test-results/<< parameters.flag >> - selection="$(bash .circleci/scripts/unit_selection.sh << parameters.flag >>)" || { echo "unit_selection.sh failed for << parameters.flag >>"; exit 1; } - [ -n "${selection}" ] || { echo "unit_selection.sh produced no files for << parameters.flag >>"; exit 1; } - shard="$(printf '%s\n' "${selection}" | circleci tests split --split-by=timings --timings-type=filename)" || { echo "circleci tests split failed for << parameters.flag >>"; exit 1; } - [ -n "${shard}" ] || { echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.flag >> files; nothing to run"; exit 0; } - mapfile -t files < <(printf '%s\n' "${shard}") - xdist_args=() - if [ "<< parameters.workers >>" -gt 0 ]; then xdist_args=(-n << parameters.workers >> --dist=<< parameters.dist >>); fi - rerun_args=(-p no:rerunfailures) - if [ "<< parameters.reruns >>" -gt 0 ]; then rerun_args=(--reruns << parameters.reruns >> --reruns-delay 1 --rerun-except "from pytest-timeout"); fi - test_env=(PATH="$PATH" HOME="$HOME" CI=true COVERAGE_CORE="$COVERAGE_CORE" LITELLM_LOCAL_MODEL_COST_MAP="$LITELLM_LOCAL_MODEL_COST_MAP") - if [ -n "${MCP_TEST_PEER_PYTHON:-}" ]; then test_env+=(MCP_TEST_PEER_PYTHON="$MCP_TEST_PEER_PYTHON"); fi - set +e - env -i "${test_env[@]}" \ - uv run --no-sync pytest "${files[@]}" "${rerun_args[@]}" -p no:pytest-retry --timeout=90 "${xdist_args[@]}" --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml - status=$? - set -e - if [ "$status" -eq 5 ]; then echo "pytest collected no tests from the shard; passing"; exit 0; fi - exit "$status" - - install_codecov_cli - - run: - name: Upload coverage - when: always - command: | - [ -f coverage.xml ] || { echo "no coverage.xml produced; skipping upload"; exit 0; } - codecov upload-process --disable-search -f coverage.xml -F << parameters.flag >> -C "$CIRCLE_SHA1" -n "<< parameters.flag >>-${CIRCLE_NODE_INDEX}-${CIRCLE_BUILD_NUM}" --git-service github - - store_test_results: - path: test-results - - store_artifacts: - path: test-results - - store_artifacts: - path: coverage.xml - documentation: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - setup_test_deps - - run: - name: Checkout litellm-docs - command: rm -rf docs/my-website && git clone --depth 1 https://github.com/BerriAI/litellm-docs.git docs/my-website - - run: - name: Run documentation validation - command: | - uv run --no-sync python ./tests/documentation_tests/test_env_keys.py - uv run --no-sync python ./tests/documentation_tests/test_router_settings.py - uv run --no-sync python ./tests/documentation_tests/test_api_docs.py - uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py - integration: - parameters: - suite: - type: string - base_ref: - type: string - default: "" - pull_request_url: - type: string - default: "" - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - skip_unless_relevant: - base_ref: << parameters.base_ref >> - pull_request_url: << parameters.pull_request_url >> - - setup_test_deps - - start_postgres: - image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - - start_redis - - run: - name: Run owned integration contracts - command: env -i PATH="$PATH" HOME="$HOME" CIRCLE_SHA1="$CIRCLE_SHA1" CIRCLE_WORKFLOW_ID="$CIRCLE_WORKFLOW_ID" bash .circleci/scripts/run_integration.sh << parameters.suite >> - no_output_timeout: 15m - - run: - name: Stop owned database and Redis - when: always - command: | - mkdir -p test-results/integration-<< parameters.suite >> - docker logs postgres-db > test-results/integration-<< parameters.suite >>/postgres.log 2>&1 || true - docker logs redis-cache > test-results/integration-<< parameters.suite >>/redis.log 2>&1 || true - docker rm -f postgres-db redis-cache - test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)" - - store_test_results: - path: test-results - - store_artifacts: - path: test-results - -workflows: - tests: - when: (pipeline.event.name == "push" and pipeline.git.branch == "main") or pipeline.event.name == "api" or (pipeline.event.name == "pull_request" and (pipeline.event.github.pull_request.base.ref == "main" or pipeline.event.github.pull_request.base.ref starts-with "litellm_")) - jobs: - - unit: - base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> - pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> - - unit: - name: unit-<< matrix.flag >> - shards: 1 - workers: 2 - reruns: 2 - matrix: - parameters: - flag: [caching-local, proxy-extras, enterprise-routing] - base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> - pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> - - unit: - name: unit-mcp-integration - flag: mcp-integration - shards: 1 - workers: 2 - legacy_mcp_peer: true - base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> - pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> - - unit: - name: unit-<< matrix.flag >> - shards: 1 - reruns: 2 - matrix: - parameters: - flag: - - enterprise-package - - proxy-infra - - responses-caching-types - - proxy-db-auth-checks - - proxy-db-jwt-and-keys - - proxy-db-proxy-server-core - - proxy-db-proxy-runtime - - proxy-db-custom-logging - - proxy-db-logging-misc - - proxy-db-db-and-spend - - proxy-db-guardrails-hooks - - proxy-db-budgets - - proxy-db-endpoints-and-responses - base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> - pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> - - unit: - name: unit-llm-vertex-ai - flag: llm-vertex-ai - shards: 2 - workers: 1 - reruns: 2 - base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> - pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> - - unit: - name: unit-llm-other-providers - flag: llm-other-providers - shards: 3 - reruns: 2 - base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> - pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> - - unit: - name: unit-core-utils - flag: core-utils - shards: 2 - reruns: 1 - base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> - pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> - - unit: - name: unit-integrations - flag: integrations - shards: 2 - reruns: 3 - base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> - pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> - - unit: - name: unit-misc - flag: misc - shards: 2 - reruns: 2 - base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> - pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> - - unit: - name: unit-proxy-db-proxy-utils - flag: proxy-db-proxy-utils - shards: 1 - reruns: 2 - dist: worksteal - base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> - pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> - - unit: - name: unit-proxy-db-key-generation - flag: proxy-db-key-generation - shards: 1 - workers: 0 - reruns: 2 - base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> - pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> - - documentation - - integration: - name: integration-<< matrix.suite >> - matrix: - parameters: - suite: [sdk] - base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> - pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> diff --git a/.github/scripts/assert_ci_coverage.py b/.github/scripts/assert_ci_coverage.py index 59511c7dc7b..e90589d3c95 100644 --- a/.github/scripts/assert_ci_coverage.py +++ b/.github/scripts/assert_ci_coverage.py @@ -9,7 +9,6 @@ import sys import warnings from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass -from types import MappingProxyType from typing import Final import yaml @@ -27,6 +26,7 @@ DOCKERFILE_INPUT_KEYS = frozenset({"file", "dockerfile"}) TEST_RUNNER_RE = re.compile(r"\bpytest\b|\bcircleci tests\b|\bhelm unittest\b|\bplaywright test\b|\bpython[0-9.]*\s") IMAGE_BUILD_RE = re.compile(r"\bdocker\s+(?:buildx\s+)?build\b") TEST_TOKEN_RE = re.compile(r"tests/[A-Za-z0-9_./*?-]+") +IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)") DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*") COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE) GLOB_CHARS = frozenset("*?") @@ -72,6 +72,17 @@ class Scalar: value: str +@dataclass(frozen=True, slots=True) +class Selection: + included: frozenset[str] + ignored: frozenset[str] + + def covers(self, relative_path: str) -> bool: + return any(_token_covers(token, relative_path) for token in self.included) and not any( + _token_covers(token, relative_path) for token in self.ignored + ) + + @dataclass(frozen=True, slots=True) class Finding: subject: str @@ -111,44 +122,32 @@ def _uncommented(value: str) -> str: return COMMENT_RE.sub("", value) -def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]: - return frozenset( - match.group(0).rstrip("/") +def _selection_for_scalar(scalar: Scalar) -> Selection: + text: Final = _uncommented(scalar.value) + ignored: Final = frozenset().union( + *( + frozenset(token.rstrip("/") for token in TEST_TOKEN_RE.findall(ignored_argument)) + for ignored_argument in IGNORE_ARG_RE.findall(text) + ) + ) + included: Final = frozenset( + match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(IGNORE_ARG_RE.sub("", text)) + ) + return Selection(included=included, ignored=ignored) + + +def _invoked_selections(scalars: Iterable[Scalar]) -> tuple[Selection, ...]: + selected_scalars: Final = tuple( + scalar for scalar in scalars if scalar.key in TEST_PATH_KEYS or TEST_RUNNER_RE.search(scalar.value) - for match in TEST_TOKEN_RE.finditer(_uncommented(scalar.value)) ) + selections: Final = tuple(_selection_for_scalar(scalar) for scalar in selected_scalars) + return tuple(selection for selection in selections if selection.included) -SELECTION_ARM_RE = re.compile(r"(?ms)^\s*([A-Za-z0-9_|*-]+)\)\s*(.*?);;") - - -def _unit_selection_arms(repo_root: pathlib.Path = REPO_ROOT) -> Mapping[str, frozenset[str]]: - script: Final = repo_root / ".circleci/scripts/unit_selection.sh" - if not script.is_file(): - return MappingProxyType({}) - text: Final = _uncommented(script.read_text()) - return MappingProxyType( - { - label: frozenset(match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(body)) - for label, body in SELECTION_ARM_RE.findall(text) - } - ) - - -def _unit_selection_tokens(repo_root: pathlib.Path = REPO_ROOT) -> frozenset[str]: - return frozenset(token for tokens in _unit_selection_arms(repo_root).values() for token in tokens) - - -def _wired_unit_flags(scalars: Iterable[Scalar]) -> frozenset[str]: - return frozenset(scalar.value for scalar in scalars if scalar.key == "unit-flag" and "${{" not in scalar.value) - - -def _shard_tokens(scalars: Iterable[Scalar], arms: Mapping[str, frozenset[str]]) -> frozenset[str]: - wired: Final = _wired_unit_flags(scalars) - return _invoked_test_tokens(scalars) | frozenset( - token for label, tokens in arms.items() if label in wired for token in tokens - ) +def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]: + return frozenset().union(*(selection.included for selection in _invoked_selections(scalars))) def _built_dockerfile_tokens(scalars: Iterable[Scalar]) -> frozenset[str]: @@ -211,11 +210,12 @@ def _dockerfiles() -> tuple[str, ...]: ) -def _uncovered_tests(allowlist: Allowlist, tokens: frozenset[str]) -> tuple[Finding, ...]: +def _uncovered_tests(allowlist: Allowlist, selections: tuple[Selection, ...]) -> tuple[Finding, ...]: uncovered = tuple( relative_path for relative_path in _test_files() - if not any(_token_covers(token, relative_path) for token in tokens) and not allowlist.covers_test(relative_path) + if not any(selection.covers(relative_path) for selection in selections) + and not allowlist.covers_test(relative_path) ) directories = tuple(dict.fromkeys(path.rsplit("/", 1)[0] for path in uncovered)) return tuple( @@ -505,7 +505,7 @@ def _check_slices() -> int: def _check_shards() -> int: - findings = _unassigned_shard_children(_shard_tokens(_all_scalars(), _unit_selection_arms())) + findings = _unassigned_shard_children(_invoked_test_tokens(_all_scalars())) if findings: _report( "test directories and files that no shard claims", @@ -678,7 +678,11 @@ def main() -> int: integration_paths, ownership_findings = _integration_ownership() test_findings = ( - _uncovered_tests(allowlist, _invoked_test_tokens(scalars) | _unit_selection_tokens() | integration_paths) + _uncovered_tests( + allowlist, + _invoked_selections(scalars) + + (Selection(included=integration_paths, ignored=frozenset()),), + ) + ownership_findings ) dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars)) diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index a5a7787c569..7db01d721cf 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -13,16 +13,6 @@ on: have its path existence-checked like any other token. required: true type: string - unit-flag: - description: >- - Codecov flag of the `.circleci/tests.yml` job that now owns part of - this shard. The shard also runs the files - `.circleci/scripts/unit_selection.sh` lists for the flag, on every - event, because the CircleCI pipeline is manual-only while the tests - migrate. - required: false - type: string - default: "" workers: description: "Number of pytest-xdist workers" required: false @@ -138,7 +128,7 @@ jobs: timeout-minutes: 8 run: | diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]' - name: Cache Prisma binaries @@ -158,7 +148,6 @@ jobs: timeout-minutes: ${{ inputs.timeout-minutes }} env: TEST_PATH: ${{ inputs.test-path }} - UNIT_FLAG: ${{ inputs.unit-flag }} MAX_FAILURES: ${{ inputs.max-failures }} WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} @@ -168,9 +157,6 @@ jobs: run: | echo "has-coverage=false" >> "$GITHUB_OUTPUT" selection="${TEST_PATH}" - if [ -n "${UNIT_FLAG}" ]; then - selection="${TEST_PATH} $(bash .circleci/scripts/unit_selection.sh "${UNIT_FLAG}" | tr '\n' ' ')" - fi if [ -z "${selection// /}" ]; then echo "shard selection is empty; nothing to run" exit 0 diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index da4477b6947..c4397c2067b 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -20,13 +20,6 @@ concurrency: # rather than alphabetical letter ranges. Adding a new test file means adding it # to whichever group it belongs to, not reshuffling slices. # -# `.circleci/tests.yml` runs each group's files on same-repo events under the -# `proxy-db-` Codecov flag; `.circleci/scripts/unit_selection.sh` holds -# the file lists. That pipeline is manual-only while the tests migrate, so -# `unit-flag` makes the shard run that list on every event. `test-path` keeps -# the files that still reach real providers and never left -# tests/proxy_unit_tests. -# # Design targets: # * Every shard runs in <= 7 minutes of wall-clock on the default runner. # Most of a shard's time is pytest plugin load + xdist worker imports + @@ -78,93 +71,136 @@ jobs: include: # Must run serially — event-loop conflict with the logging worker. - test-group: key-generation - test-path: "" - unit-flag: proxy-db-key-generation + test-path: >- + tests/unit/proxy/management_endpoints/test_key_generate_prisma.py workers: 0 dist: loadscope timeout: 20 # ---- auth: split into 2 shards ---- - test-group: auth-checks - test-path: "" - unit-flag: proxy-db-auth-checks + test-path: >- + tests/unit/proxy/auth/test_auth_checks.py + tests/unit/proxy/auth/test_user_api_key_auth.py + tests/unit/proxy/test_credential_slot_registry.py + tests/unit/proxy/test_deprecated_key_grace_period.py workers: 4 dist: loadscope timeout: 15 - test-group: jwt-and-keys - test-path: "" - unit-flag: proxy-db-jwt-and-keys + test-path: >- + tests/unit/proxy/auth/test_jwt.py + tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py + tests/unit/proxy/test_proxy_custom_auth.py workers: 4 dist: loadscope timeout: 15 # ---- test_proxy_utils.py, single shard, worksteal distribution ---- - test-group: proxy-utils - test-path: "" - unit-flag: proxy-db-proxy-utils + test-path: >- + tests/unit/proxy/test_proxy_utils.py workers: 4 dist: worksteal timeout: 15 # ---- proxy server: split into 2 shards ---- - test-group: proxy-server-core - test-path: "tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py" - unit-flag: proxy-db-proxy-server-core + test-path: >- + tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py + tests/unit/proxy/test__lazy_features.py + tests/unit/proxy/test_aproxy_startup.py + tests/unit/proxy/test_proxy_server.py workers: 4 dist: loadscope timeout: 15 - test-group: proxy-runtime - test-path: "" - unit-flag: proxy-db-proxy-runtime + test-path: >- + tests/unit/proxy/auth/test_multipart_bypass_repro.py + tests/unit/proxy/auth/test_proxy_routes.py + tests/unit/proxy/middleware/test_request_size_limit_middleware.py + tests/unit/proxy/test_proxy_config_unit_test.py + tests/unit/proxy/test_proxy_token_counter.py + tests/unit/proxy/test_server_root_path.py workers: 4 dist: loadscope timeout: 15 # ---- logging: split into 2 shards ---- - test-group: custom-logging - test-path: "tests/proxy_unit_tests/test_proxy_custom_logger.py" - unit-flag: proxy-db-custom-logging + test-path: >- + tests/proxy_unit_tests/test_proxy_custom_logger.py + tests/unit/proxy/test_custom_callback_input.py + tests/unit/proxy/test_custom_logger_s3_gcs.py workers: 4 dist: loadscope timeout: 15 - test-group: logging-misc - test-path: "" - unit-flag: proxy-db-logging-misc + test-path: >- + tests/unit/proxy/management_helpers/test_audit_logs_proxy.py + tests/unit/proxy/spend_tracking/test_search_api_logging.py + tests/unit/proxy/test_proxy_reject_logging.py workers: 4 dist: loadscope timeout: 15 - test-group: db-and-spend - test-path: "" - unit-flag: proxy-db-db-and-spend + test-path: >- + tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py + tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py + tests/unit/proxy/db/test_update_daily_tag_spend.py + tests/unit/proxy/test_db_schema_changes.py + tests/unit/proxy/test_prisma_client_backoff_retry.py + tests/unit/proxy/test_update_spend.py + tests/unit/skills/test_skills_db.py workers: 4 dist: loadscope timeout: 15 # ---- guardrails + budget + hooks: split into 2 ---- - test-group: guardrails-hooks - test-path: "" - unit-flag: proxy-db-guardrails-hooks + test-path: >- + tests/unit/proxy/hooks/test_banned_keyword_list.py + tests/unit/proxy/test_proxy_setting_guardrails.py + tests/unit/proxy/test_unit_test_proxy_hooks.py workers: 4 dist: loadscope timeout: 15 - test-group: budgets - test-path: "" - unit-flag: proxy-db-budgets + test-path: >- + tests/unit/proxy/auth/test_default_end_user_budget_simple.py + tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py + tests/unit/proxy/test_zero_cost_model_budget_bypass.py workers: 4 dist: loadscope timeout: 15 - test-group: endpoints-and-responses - test-path: "tests/proxy_unit_tests/test_proxy_exception_mapping.py" - unit-flag: proxy-db-endpoints-and-responses + test-path: >- + tests/proxy_unit_tests/test_proxy_exception_mapping.py + tests/unit/proxy/lens + tests/unit/proxy/auth/test_models_fallback_endpoint.py + tests/unit/proxy/common_utils/test_check_batch_cost.py + tests/unit/proxy/common_utils/test_check_responses_cost.py + tests/unit/proxy/common_utils/test_realtime_cache.py + tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py + tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py + tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py + tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py + tests/unit/proxy/response_polling + tests/unit/proxy/test_custom_tokenizer_bug.py + tests/unit/proxy/test_get_favicon.py + tests/unit/proxy/test_get_image.py + tests/unit/proxy/test_prompt_test_endpoint.py + tests/unit/proxy/test_reducto_ocr_route.py + tests/unit/proxy/test_response_polling_pre_call_checks.py + tests/unit/proxy/test_ui_path_detection.py workers: 4 dist: loadscope timeout: 15 uses: ./.github/workflows/_test-unit-base.yml with: test-path: ${{ matrix.test-path }} - unit-flag: ${{ matrix.unit-flag }} workers: ${{ matrix.workers }} reruns: 2 timeout-minutes: ${{ matrix.timeout }} diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 3f99cee1720..5f9ea430069 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -35,10 +35,6 @@ concurrency: # a matrix and carries a shard-coverage guard that reads that file by name. # Folding it in here is a follow-up, together with generalising that guard into # assert_ci_coverage.py. -# -# `unit-flag` names the `.circleci/tests.yml` job that now runs part of the -# shard under the same Codecov flag. That pipeline is manual-only while the -# tests migrate, so the shard also runs those files on every event. jobs: unit: name: ${{ matrix.shard }} @@ -52,8 +48,9 @@ jobs: include: - shard: core-utils artifact-name: core-utils - test-path: tests/unit/decisions - unit-flag: core-utils + test-path: >- + tests/unit/decisions + tests/unit/litellm_core_utils workers: 2 reruns: 1 timeout-minutes: 20 @@ -61,8 +58,22 @@ jobs: - shard: enterprise-routing artifact-name: enterprise-routing - test-path: "" - unit-flag: enterprise-routing + test-path: >- + tests/unit/google_genai + tests/unit/router_strategy + tests/unit/router_utils + tests/unit/proxy/common_utils/test_cache_aware_routing.py + tests/unit/enterprise/enterprise_callbacks/send_emails + tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py + tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py + tests/unit/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py + tests/unit/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py + tests/unit/enterprise/proxy/test_batch_update_db_managed_output_file_id.py + tests/unit/enterprise/proxy/test_deleted_file_returns_403_not_404.py + tests/unit/enterprise/proxy/test_enterprise_routes.py + tests/unit/enterprise/proxy/test_file_deletion_blocking.py + tests/unit/enterprise/proxy/test_managed_files_access_check.py + tests/unit/enterprise/proxy/test_managed_files_hook.py workers: 2 reruns: 2 timeout-minutes: 20 @@ -73,7 +84,7 @@ jobs: test-path: >- tests/test_litellm/integrations tests/test_litellm/tracing - unit-flag: integrations + tests/unit/integrations workers: 2 reruns: 3 timeout-minutes: 20 @@ -81,8 +92,8 @@ jobs: - shard: Vertex AI artifact-name: llm-vertex-ai - test-path: "" - unit-flag: llm-vertex-ai + test-path: >- + tests/unit/llms/vertex_ai workers: 1 reruns: 2 timeout-minutes: 20 @@ -90,8 +101,10 @@ jobs: - shard: All Other Providers artifact-name: llm-other-providers - test-path: "" - unit-flag: llm-other-providers + test-path: >- + tests/unit/llms + --ignore=tests/unit/llms/vertex_ai + --ignore=tests/unit/llms/base_llm/batches/base_batches_config_test.py workers: 2 reruns: 2 timeout-minutes: 20 @@ -101,7 +114,27 @@ jobs: artifact-name: misc test-path: >- tests/test_litellm/test_*.py - unit-flag: misc + tests/unit/test_*.py + tests/unit/test_router + tests/unit/a2a_protocol + tests/unit/batches + tests/unit/chat_completions + tests/unit/completion_extras + tests/unit/containers + tests/unit/embeddings + tests/unit/endpoints + tests/unit/files + tests/unit/harness + tests/unit/images + tests/unit/interactions + tests/unit/messages + tests/unit/rag + tests/unit/rerank_api + tests/unit/rust_bridge + tests/unit/secret_managers + tests/unit/vector_stores + tests/unit/videos + --ignore=tests/unit/rust_bridge/native_route_wheel_test.py workers: 2 reruns: 2 timeout-minutes: 20 @@ -209,7 +242,10 @@ jobs: tests/unit/proxy/enterprise_billing tests/unit/proxy/types_utils tests/unit/proxy/logging_endpoints - unit-flag: proxy-infra + tests/unit/gateway + tests/unit/proxy/management + tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py + tests/unit/proxy/roi_calculator workers: 4 reruns: 2 timeout-minutes: 20 @@ -251,8 +287,8 @@ jobs: - shard: caching-local artifact-name: caching-local - test-path: "" - unit-flag: caching-local + test-path: >- + tests/unit/caching workers: 2 reruns: 2 timeout-minutes: 20 @@ -260,8 +296,8 @@ jobs: - shard: proxy-extras artifact-name: proxy-extras - test-path: "" - unit-flag: proxy-extras + test-path: >- + tests/unit/litellm_proxy_extras workers: 2 reruns: 2 timeout-minutes: 20 @@ -269,8 +305,15 @@ jobs: - shard: enterprise-package artifact-name: enterprise-package - test-path: "" - unit-flag: enterprise-package + test-path: >- + tests/unit/enterprise/integrations + tests/unit/enterprise/proxy/auth + tests/unit/enterprise/proxy/guardrails + tests/unit/enterprise/proxy/hooks + tests/unit/enterprise/proxy/management_endpoints + tests/unit/enterprise/proxy/test_audit_logging_endpoints.py + tests/unit/enterprise/proxy/test_liteadmin.py + tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py workers: 4 reruns: 2 timeout-minutes: 20 @@ -278,16 +321,39 @@ jobs: - shard: responses-caching-types artifact-name: responses-caching-types - test-path: "" - unit-flag: responses-caching-types + test-path: >- + tests/unit/responses + tests/unit/types + --ignore=tests/unit/responses/mcp workers: 2 reruns: 2 timeout-minutes: 20 job-timeout-minutes: 60 + + - shard: unit + artifact-name: unit + test-path: >- + tests/unit/anthropic_interface + tests/unit/compression + tests/unit/enterprise/enterprise_callbacks/test_callback_controls.py + tests/unit/enterprise/enterprise_callbacks/test_llm_guard.py + tests/unit/enterprise/enterprise_callbacks/test_secret_detection.py + tests/unit/integration_support + tests/unit/models + tests/unit/ocr + tests/unit/passthrough + tests/unit/realtime_api + tests/unit/repositories + tests/unit/sandbox + tests/unit/skills/test_skills_main.py + tests/unit/tracing + workers: 2 + reruns: 0 + timeout-minutes: 20 + job-timeout-minutes: 60 uses: ./.github/workflows/_test-unit-base.yml with: test-path: ${{ matrix.test-path }} - unit-flag: ${{ matrix.unit-flag || '' }} workers: ${{ matrix.workers }} reruns: ${{ matrix.reruns }} timeout-minutes: ${{ matrix.timeout-minutes }} diff --git a/tests/unit/test_assert_ci_coverage.py b/tests/unit/test_assert_ci_coverage.py index e4233dbe682..4498dbe1498 100644 --- a/tests/unit/test_assert_ci_coverage.py +++ b/tests/unit/test_assert_ci_coverage.py @@ -227,37 +227,39 @@ def test_every_sharded_root_named_in_the_script_exists_on_disk(): def test_the_repo_as_it_stands_has_every_shard_child_assigned(): findings = coverage._unassigned_shard_children( - coverage._shard_tokens(coverage._all_scalars(), coverage._unit_selection_arms()) + coverage._invoked_test_tokens(coverage._all_scalars()) ) assert [f.subject for f in findings] == [] -def test_shard_tokens_credits_only_wired_unit_flags(tmp_path): - root = tmp_path / "tests" / "tree" - (root / "wired").mkdir(parents=True) - (root / "wired" / "test_a.py").write_text("def test_a(): assert True\n") - (root / "unwired").mkdir(parents=True) - (root / "unwired" / "test_b.py").write_text("def test_b(): assert True\n") - script = tmp_path / ".circleci" / "scripts" / "unit_selection.sh" - script.parent.mkdir(parents=True) - script.write_text( - "legacy_paths() {\n" - " case \"$1\" in\n" - " wired-flag) echo tests/tree/wired ;;\n" - " unwired-flag)\n" - " echo tests/tree/unwired ;;\n" - " esac\n" - "}\n" +def test_shards_are_credited_only_from_explicit_test_paths() -> None: + scalars: Final = ( + coverage.Scalar(key="test-path", value="tests/unit/wired\ntests/unit/also_wired"), + coverage.Scalar(key="shard", value="tests/unit/not_a_test_path"), ) + assert coverage._invoked_test_tokens(scalars) == frozenset({"tests/unit/wired", "tests/unit/also_wired"}) - scalars: Final = (coverage.Scalar(key="unit-flag", value="wired-flag"),) - findings = coverage._unassigned_shard_children( - coverage._shard_tokens(scalars, coverage._unit_selection_arms(tmp_path)), - roots=("tests/tree",), - repo_root=tmp_path, + +def test_an_ignored_path_is_not_credited_as_invoked() -> None: + scalars: Final = ( + coverage.Scalar(key="test-path", value="tests/unit/a\n--ignore=tests/unit/b/test_x.py"), ) + assert coverage._invoked_test_tokens(scalars) == frozenset({"tests/unit/a"}) - assert tuple(f.subject for f in findings) == ("tests/tree/unwired",) + +def test_a_file_its_only_shard_ignores_is_not_covered_by_that_shards_glob() -> None: + selections: Final = coverage._invoked_selections( + ( + coverage.Scalar( + key="test-path", + value="tests/unit/proxy/test_*.py --ignore=tests/unit/proxy/test_update_spend.py", + ), + ) + ) + assert len(selections) == 1 + selection: Final = selections[0] + assert selection.covers("tests/unit/proxy/test_other.py") is True + assert selection.covers("tests/unit/proxy/test_update_spend.py") is False def test_check_shards_passes_on_the_repo_as_it_stands(capsys): diff --git a/tests/unit/test_unit_shard_missing_paths.py b/tests/unit/test_unit_shard_missing_paths.py index 0360a227142..4fa9c5bd3c1 100644 --- a/tests/unit/test_unit_shard_missing_paths.py +++ b/tests/unit/test_unit_shard_missing_paths.py @@ -38,7 +38,6 @@ def _run_shard(tmp_path: Path, test_path: str, workers: str) -> subprocess.Compl "PATH": f"{shim_dir}{os.pathsep}{os.environ['PATH']}", "GITHUB_OUTPUT": str(tmp_path / "github_output"), "TEST_PATH": test_path, - "UNIT_FLAG": "", "WORKERS": workers, }, capture_output=True,