diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index 26a2c427f32..8a3a9107b51 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -89,6 +89,9 @@ jobs: - name: test_workflow_job_name_collisions run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_workflow_job_name_collisions.py + - name: test_unit_passed_gate + run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_unit_passed_gate.py + - name: test_e2e_changed_gate run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml deleted file mode 100644 index 3a989dc6651..00000000000 --- a/.github/workflows/test-unit-proxy-db.yml +++ /dev/null @@ -1,233 +0,0 @@ -name: "Unit Tests: Proxy DB Operations" - -on: - pull_request: - branches: - - main - - "litellm_**" - push: - branches: - - main - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -# Semantic matrix: each shard groups tests by concern (auth, server, logging, …) -# rather than alphabetical letter ranges. Adding a new test file means adding it -# to whichever group it belongs to, not reshuffling slices. -# -# Design targets: -# * Every shard runs in <= 7 minutes of wall-clock on the default runner. -# Most of a shard's time is pytest plugin load + xdist worker imports + -# pytest-cov instrumentation, not the tests themselves. Keeping per-shard -# work low and matching worker count to runner cores is what controls it. -# * `timeout` bounds the pytest step only. Checkout, dependency install, and -# Prisma client generation draw on a separate allowance in the base -# workflow, so slow setup shows up as a slow job rather than as a -# cancelled shard whose tests were passing. -# * workers: 4 matches the 4-core ubuntu-latest runner. -n 8 on 4 cores -# oversubscribes 2x and workers fight for CPU during their cold-start -# imports (measured ~441% CPU for -n 8 locally, i.e. ~55% effective). -# * test_key_generate_prisma.py stays serial (workers=0) — it has event-loop -# conflicts with the logging worker when run in parallel. -# * test_proxy_utils.py runs as a single shard with --dist=worksteal so -# xdist balances its 188 parametrized cases across workers instead of -# pinning the whole file to one worker (the default --dist=loadscope -# behavior for single-file targets). -jobs: - lens-python-310: - name: Lens Python 3.10 - permissions: - contents: read - id-token: write - pull-requests: write - uses: ./.github/workflows/_test-unit-base.yml - with: - python-version: "3.10" - test-path: tests/unit/proxy/lens/test_inference.py - workers: 0 - reruns: 0 - timeout-minutes: 5 - artifact-name: lens-python-310 - - # Fast guard — fails the workflow when a test directory or file inside a sharded - # tree is claimed by no shard. The semantic-shard design has no catch-all bucket, - # so an unassigned child runs nowhere; assert_ci_coverage.py holds the tree list - # and reads the same test-path keys the coverage census does. - assert-shard-coverage: - runs-on: ubuntu-latest - timeout-minutes: 2 - permissions: - contents: read - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - name: Assert every test directory and file is claimed by a shard - run: python3 .github/scripts/assert_ci_coverage.py --shards - - proxy-db: - needs: assert-shard-coverage - # Display only the semantic shard name in the checks UI instead of GHA's - # default "proxy-db (key-generation, tests/unit/proxy/…, 0, loadscope, 20)" - # which includes every matrix field and gets truncated past the test-path. - name: ${{ matrix.test-group }} - permissions: - contents: read - id-token: write - pull-requests: write - strategy: - fail-fast: false - matrix: - include: - # Must run serially — event-loop conflict with the logging worker. - - test-group: key-generation - test-path: >- - tests/unit/proxy/management_endpoints/test_key_generate_prisma.py - workers: 0 - dist: loadscope - timeout: 20 - - # ---- auth: split into 2 shards ---- - - test-group: auth-checks - test-path: >- - tests/unit/proxy/auth/test_auth_checks.py - tests/unit/proxy/auth/test_user_api_key_auth.py - tests/unit/proxy/test_credential_slot_registry.py - tests/unit/proxy/test_deprecated_key_grace_period.py - workers: 4 - dist: loadscope - timeout: 15 - - test-group: jwt-and-keys - test-path: >- - tests/unit/proxy/auth/test_jwt.py - tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py - tests/unit/proxy/test_proxy_custom_auth.py - workers: 4 - dist: loadscope - timeout: 15 - - # ---- test_proxy_utils.py, single shard, worksteal distribution ---- - - test-group: proxy-utils - test-path: >- - tests/unit/proxy/test_proxy_utils.py - workers: 4 - dist: worksteal - timeout: 15 - - # ---- proxy server: split into 2 shards ---- - - test-group: proxy-server-core - test-path: >- - tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py - tests/unit/proxy/test__lazy_features.py - tests/unit/proxy/test_aproxy_startup.py - tests/unit/proxy/test_proxy_server.py - workers: 4 - dist: loadscope - timeout: 15 - - test-group: proxy-runtime - test-path: >- - tests/unit/proxy/auth/test_multipart_bypass_repro.py - tests/unit/proxy/auth/test_proxy_routes.py - tests/unit/proxy/middleware/test_request_size_limit_middleware.py - tests/unit/proxy/test_proxy_config_unit_test.py - tests/unit/proxy/test_proxy_token_counter.py - tests/unit/proxy/test_server_root_path.py - workers: 4 - dist: loadscope - timeout: 15 - - - test-group: mcp-oauth - test-path: >- - tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py - tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py - tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py - tests/unit/proxy/_experimental/mcp_server/outbound_credentials - workers: 4 - dist: loadscope - timeout: 15 - - # ---- logging: split into 2 shards ---- - - test-group: custom-logging - test-path: >- - tests/proxy_unit_tests/test_proxy_custom_logger.py - tests/unit/proxy/test_custom_callback_input.py - tests/unit/proxy/test_custom_logger_s3_gcs.py - workers: 4 - dist: loadscope - timeout: 15 - - test-group: logging-misc - test-path: >- - tests/unit/proxy/management_helpers/test_audit_logs_proxy.py - tests/unit/proxy/spend_tracking/test_search_api_logging.py - tests/unit/proxy/test_proxy_reject_logging.py - workers: 4 - dist: loadscope - timeout: 15 - - - test-group: db-and-spend - test-path: >- - tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py - tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py - tests/unit/proxy/db/test_update_daily_tag_spend.py - tests/unit/proxy/test_db_schema_changes.py - tests/unit/proxy/test_prisma_client_backoff_retry.py - tests/unit/proxy/test_update_spend.py - tests/unit/skills/test_skills_db.py - workers: 4 - dist: loadscope - timeout: 15 - - # ---- guardrails + budget + hooks: split into 2 ---- - - test-group: guardrails-hooks - test-path: >- - tests/unit/proxy/hooks/test_banned_keyword_list.py - tests/unit/proxy/test_proxy_setting_guardrails.py - tests/unit/proxy/test_unit_test_proxy_hooks.py - workers: 4 - dist: loadscope - timeout: 15 - - test-group: budgets - test-path: >- - tests/unit/proxy/auth/test_default_end_user_budget_simple.py - tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py - tests/unit/proxy/test_zero_cost_model_budget_bypass.py - workers: 4 - dist: loadscope - timeout: 15 - - - test-group: endpoints-and-responses - test-path: >- - tests/proxy_unit_tests/test_proxy_exception_mapping.py - tests/unit/proxy/lens - tests/unit/proxy/auth/test_models_fallback_endpoint.py - tests/unit/proxy/common_utils/test_check_batch_cost.py - tests/unit/proxy/common_utils/test_check_responses_cost.py - tests/unit/proxy/common_utils/test_realtime_cache.py - tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py - tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py - tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py - tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py - tests/unit/proxy/response_polling - tests/unit/proxy/test_custom_tokenizer_bug.py - tests/unit/proxy/test_get_favicon.py - tests/unit/proxy/test_get_image.py - tests/unit/proxy/test_prompt_test_endpoint.py - tests/unit/proxy/test_reducto_ocr_route.py - tests/unit/proxy/test_response_polling_pre_call_checks.py - tests/unit/proxy/test_ui_path_detection.py - workers: 4 - dist: loadscope - timeout: 15 - uses: ./.github/workflows/_test-unit-base.yml - with: - test-path: ${{ matrix.test-path }} - workers: ${{ matrix.workers }} - reruns: 2 - timeout-minutes: ${{ matrix.timeout }} - dist: ${{ matrix.dist }} - artifact-name: proxy-db-${{ matrix.test-group }} diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index ce4f5b52f4e..21b82db44b7 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -28,11 +28,6 @@ concurrency: # defaults. An absent matrix key renders as an empty string, which is not a # number, so a partially-specified entry would fail the call rather than fall # back to the default. -# -# tests/unit/proxy keeps its own caller (test-unit-proxy-db.yml): it is already -# a matrix and carries a shard-coverage guard that reads that file by name. -# Folding it in here is a follow-up, together with generalising that guard into -# assert_ci_coverage.py. jobs: rust-bridge: name: Build the Rust bridge @@ -508,3 +503,211 @@ jobs: job-timeout-minutes: ${{ matrix.job-timeout-minutes }} dist: ${{ matrix.dist || 'loadscope' }} artifact-name: ${{ matrix.artifact-name }} + + lens-python-310: + name: Lens Python 3.10 + permissions: + contents: read + id-token: write + pull-requests: write + uses: ./.github/workflows/_test-unit-base.yml + with: + python-version: "3.10" + test-path: tests/unit/proxy/lens/test_inference.py + workers: 0 + reruns: 0 + timeout-minutes: 5 + artifact-name: lens-python-310 + + # Fast guard — fails the workflow when a test directory or file inside a sharded + # tree is claimed by no shard. The semantic-shard design has no catch-all bucket, + # so an unassigned child runs nowhere; assert_ci_coverage.py holds the tree list + # and reads the same test-path keys the coverage census does. + assert-shard-coverage: + runs-on: ubuntu-latest + timeout-minutes: 2 + permissions: + contents: read + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + - name: Assert every test directory and file is claimed by a shard + run: python3 .github/scripts/assert_ci_coverage.py --shards + + proxy-db: + needs: assert-shard-coverage + # Display only the semantic shard name in the checks UI instead of GHA's + # default "proxy-db (key-generation, tests/unit/proxy/…, 0, loadscope, 20)" + # which includes every matrix field and gets truncated past the test-path. + name: ${{ matrix.test-group }} + permissions: + contents: read + id-token: write + pull-requests: write + strategy: + fail-fast: false + matrix: + include: + # Must run serially — event-loop conflict with the logging worker. + - test-group: key-generation + test-path: >- + tests/unit/proxy/management_endpoints/test_key_generate_prisma.py + workers: 0 + dist: loadscope + timeout: 20 + + # ---- auth: split into 2 shards ---- + - test-group: auth-checks + test-path: >- + tests/unit/proxy/auth/test_auth_checks.py + tests/unit/proxy/auth/test_user_api_key_auth.py + tests/unit/proxy/test_credential_slot_registry.py + tests/unit/proxy/test_deprecated_key_grace_period.py + workers: 4 + dist: loadscope + timeout: 15 + - test-group: jwt-and-keys + test-path: >- + tests/unit/proxy/auth/test_jwt.py + tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py + tests/unit/proxy/test_proxy_custom_auth.py + workers: 4 + dist: loadscope + timeout: 15 + + # ---- test_proxy_utils.py, single shard, worksteal distribution ---- + - test-group: proxy-utils + test-path: >- + tests/unit/proxy/test_proxy_utils.py + workers: 4 + dist: worksteal + timeout: 15 + + # ---- proxy server: split into 2 shards ---- + - test-group: proxy-server-core + test-path: >- + tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py + tests/unit/proxy/test__lazy_features.py + tests/unit/proxy/test_aproxy_startup.py + tests/unit/proxy/test_proxy_server.py + workers: 4 + dist: loadscope + timeout: 15 + - test-group: proxy-runtime + test-path: >- + tests/unit/proxy/auth/test_multipart_bypass_repro.py + tests/unit/proxy/auth/test_proxy_routes.py + tests/unit/proxy/middleware/test_request_size_limit_middleware.py + tests/unit/proxy/test_proxy_config_unit_test.py + tests/unit/proxy/test_proxy_token_counter.py + tests/unit/proxy/test_server_root_path.py + workers: 4 + dist: loadscope + timeout: 15 + + - test-group: mcp-oauth + test-path: >- + tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py + tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py + tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py + tests/unit/proxy/_experimental/mcp_server/outbound_credentials + workers: 4 + dist: loadscope + timeout: 15 + + # ---- logging: split into 2 shards ---- + - test-group: custom-logging + test-path: >- + tests/proxy_unit_tests/test_proxy_custom_logger.py + tests/unit/proxy/test_custom_callback_input.py + tests/unit/proxy/test_custom_logger_s3_gcs.py + workers: 4 + dist: loadscope + timeout: 15 + - test-group: logging-misc + test-path: >- + tests/unit/proxy/management_helpers/test_audit_logs_proxy.py + tests/unit/proxy/spend_tracking/test_search_api_logging.py + tests/unit/proxy/test_proxy_reject_logging.py + workers: 4 + dist: loadscope + timeout: 15 + + - test-group: db-and-spend + test-path: >- + tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py + tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py + tests/unit/proxy/db/test_update_daily_tag_spend.py + tests/unit/proxy/test_db_schema_changes.py + tests/unit/proxy/test_prisma_client_backoff_retry.py + tests/unit/proxy/test_update_spend.py + tests/unit/skills/test_skills_db.py + workers: 4 + dist: loadscope + timeout: 15 + + # ---- guardrails + budget + hooks: split into 2 ---- + - test-group: guardrails-hooks + test-path: >- + tests/unit/proxy/hooks/test_banned_keyword_list.py + tests/unit/proxy/test_proxy_setting_guardrails.py + tests/unit/proxy/test_unit_test_proxy_hooks.py + workers: 4 + dist: loadscope + timeout: 15 + - test-group: budgets + test-path: >- + tests/unit/proxy/auth/test_default_end_user_budget_simple.py + tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py + tests/unit/proxy/test_zero_cost_model_budget_bypass.py + workers: 4 + dist: loadscope + timeout: 15 + + - test-group: endpoints-and-responses + test-path: >- + tests/proxy_unit_tests/test_proxy_exception_mapping.py + tests/unit/proxy/lens + tests/unit/proxy/auth/test_models_fallback_endpoint.py + tests/unit/proxy/common_utils/test_check_batch_cost.py + tests/unit/proxy/common_utils/test_check_responses_cost.py + tests/unit/proxy/common_utils/test_realtime_cache.py + tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py + tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py + tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py + tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py + tests/unit/proxy/response_polling + tests/unit/proxy/test_custom_tokenizer_bug.py + tests/unit/proxy/test_get_favicon.py + tests/unit/proxy/test_get_image.py + tests/unit/proxy/test_prompt_test_endpoint.py + tests/unit/proxy/test_reducto_ocr_route.py + tests/unit/proxy/test_response_polling_pre_call_checks.py + tests/unit/proxy/test_ui_path_detection.py + workers: 4 + dist: loadscope + timeout: 15 + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: ${{ matrix.test-path }} + workers: ${{ matrix.workers }} + reruns: 2 + timeout-minutes: ${{ matrix.timeout }} + dist: ${{ matrix.dist }} + artifact-name: proxy-db-${{ matrix.test-group }} + + unit-passed: + name: unit passed + needs: [rust-bridge, unit, lens-python-310, assert-shard-coverage, proxy-db] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 2 + permissions: {} + steps: + - name: Require every unit job to succeed + env: + NEEDS: ${{ toJSON(needs) }} + run: | + jq -r 'to_entries[] | "\(.key): \(.value.result)"' <<< "$NEEDS" + jq -e 'all(.[]; .result == "success")' <<< "$NEEDS" > /dev/null diff --git a/tests/code_coverage_tests/test_unit_passed_gate.py b/tests/code_coverage_tests/test_unit_passed_gate.py new file mode 100644 index 00000000000..7b511538d40 --- /dev/null +++ b/tests/code_coverage_tests/test_unit_passed_gate.py @@ -0,0 +1,68 @@ +import json +import os +import subprocess +from pathlib import Path +from typing import Final + +import pytest +import yaml + +_UNIT_WORKFLOW: Final = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "test-unit.yml" +_GATE_JOB: Final = "unit-passed" + + +def _jobs() -> dict[str, dict[str, object]]: + return yaml.safe_load(_UNIT_WORKFLOW.read_text())["jobs"] + + +def _gate_script() -> str: + steps: Final = _jobs()[_GATE_JOB]["steps"] + assert isinstance(steps, list) and len(steps) == 1 + return steps[0]["run"] + + +def _run_gate(results: dict[str, str]) -> subprocess.CompletedProcess[str]: + needs: Final = {job: {"result": result, "outputs": {}} for job, result in results.items()} + return subprocess.run( + ("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", _gate_script()), + env={**os.environ, "NEEDS": json.dumps(needs)}, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + +def _needed_jobs() -> tuple[str, ...]: + needs: Final = _jobs()[_GATE_JOB]["needs"] + assert isinstance(needs, list) + return tuple(needs) + + +def test_the_gate_passes_when_every_needed_job_succeeded() -> None: + jobs: Final = _needed_jobs() + assert jobs + + result: Final = _run_gate(dict.fromkeys(jobs, "success")) + + assert result.returncode == 0, result.stdout + result.stderr + assert all(f"{job}: success" in result.stdout for job in jobs), result.stdout + + +@pytest.mark.parametrize("outcome", ("failure", "cancelled", "skipped")) +def test_the_gate_fails_when_any_needed_job_did_not_succeed(outcome: str) -> None: + jobs: Final = _needed_jobs() + assert len(jobs) > 1 + + result: Final = _run_gate({**dict.fromkeys(jobs, "success"), jobs[-1]: outcome}) + + assert result.returncode != 0 + assert f"{jobs[-1]}: {outcome}" in result.stdout, result.stdout + + +def test_the_gate_waits_for_every_other_job_and_reports_even_when_they_fail() -> None: + jobs: Final = _jobs() + gate: Final = jobs[_GATE_JOB] + + assert set(_needed_jobs()) == set(jobs) - {_GATE_JOB} + assert gate["if"] == "always()"