scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 109

ci(unit): add unit passed collector job and fold proxy-db shards into test-unit.yml (#45466)

Browse source 
* ci(unit): add unit passed collector job and fold proxy-db shards into test-unit.yml

* test(ci): cover the unit passed gate's success, failure, cancelled and skipped results

* test(ci): run the unit passed gate test from the code quality workflow

---------

Co-authored-by: yuneng <yuneng@berri.ai>
This commit is contained in:
devin-ai-integration[bot] 2026-10-08 17:24:24 -07:00 • committed by GitHub
parent acab4761a2
commit 1a42c5abde
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 279 additions and 238 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/test-code-quality.yml vendored
View file

@ -89,6 +89,9 @@ jobs:
- name: test_workflow_job_name_collisions
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_workflow_job_name_collisions.py
- name: test_unit_passed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_unit_passed_gate.py
- name: test_e2e_changed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py

233
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -1,233 +0,0 @@
name: "Unit Tests: Proxy DB Operations"
on:
pull_request:
branches:
- main
- "litellm_**"
push:
branches:
- main
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# Semantic matrix: each shard groups tests by concern (auth, server, logging, …)
# rather than alphabetical letter ranges. Adding a new test file means adding it
# to whichever group it belongs to, not reshuffling slices.
#
# Design targets:
# * Every shard runs in <= 7 minutes of wall-clock on the default runner.
# Most of a shard's time is pytest plugin load + xdist worker imports +
# pytest-cov instrumentation, not the tests themselves. Keeping per-shard
# work low and matching worker count to runner cores is what controls it.
# * `timeout` bounds the pytest step only. Checkout, dependency install, and
# Prisma client generation draw on a separate allowance in the base
# workflow, so slow setup shows up as a slow job rather than as a
# cancelled shard whose tests were passing.
# * workers: 4 matches the 4-core ubuntu-latest runner. -n 8 on 4 cores
# oversubscribes 2x and workers fight for CPU during their cold-start
# imports (measured ~441% CPU for -n 8 locally, i.e. ~55% effective).
# * test_key_generate_prisma.py stays serial (workers=0) — it has event-loop
# conflicts with the logging worker when run in parallel.
# * test_proxy_utils.py runs as a single shard with --dist=worksteal so
# xdist balances its 188 parametrized cases across workers instead of
# pinning the whole file to one worker (the default --dist=loadscope
# behavior for single-file targets).
jobs:
lens-python-310:
name: Lens Python 3.10
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
python-version: "3.10"
test-path: tests/unit/proxy/lens/test_inference.py
workers: 0
reruns: 0
timeout-minutes: 5
artifact-name: lens-python-310
# Fast guard — fails the workflow when a test directory or file inside a sharded
# tree is claimed by no shard. The semantic-shard design has no catch-all bucket,
# so an unassigned child runs nowhere; assert_ci_coverage.py holds the tree list
# and reads the same test-path keys the coverage census does.
assert-shard-coverage:
runs-on: ubuntu-latest
timeout-minutes: 2
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Assert every test directory and file is claimed by a shard
run: python3 .github/scripts/assert_ci_coverage.py --shards
proxy-db:
needs: assert-shard-coverage
# Display only the semantic shard name in the checks UI instead of GHA's
# default "proxy-db (key-generation, tests/unit/proxy/…, 0, loadscope, 20)"
# which includes every matrix field and gets truncated past the test-path.
name: ${{ matrix.test-group }}
permissions:
contents: read
id-token: write
pull-requests: write
strategy:
fail-fast: false
matrix:
include:
# Must run serially — event-loop conflict with the logging worker.
- test-group: key-generation
test-path: >-
tests/unit/proxy/management_endpoints/test_key_generate_prisma.py
workers: 0
dist: loadscope
timeout: 20
# ---- auth: split into 2 shards ----
- test-group: auth-checks
test-path: >-
tests/unit/proxy/auth/test_auth_checks.py
tests/unit/proxy/auth/test_user_api_key_auth.py
tests/unit/proxy/test_credential_slot_registry.py
tests/unit/proxy/test_deprecated_key_grace_period.py
workers: 4
dist: loadscope
timeout: 15
- test-group: jwt-and-keys
test-path: >-
tests/unit/proxy/auth/test_jwt.py
tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
tests/unit/proxy/test_proxy_custom_auth.py
workers: 4
dist: loadscope
timeout: 15
# ---- test_proxy_utils.py, single shard, worksteal distribution ----
- test-group: proxy-utils
test-path: >-
tests/unit/proxy/test_proxy_utils.py
workers: 4
dist: worksteal
timeout: 15
# ---- proxy server: split into 2 shards ----
- test-group: proxy-server-core
test-path: >-
tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py
tests/unit/proxy/test__lazy_features.py
tests/unit/proxy/test_aproxy_startup.py
tests/unit/proxy/test_proxy_server.py
workers: 4
dist: loadscope
timeout: 15
- test-group: proxy-runtime
test-path: >-
tests/unit/proxy/auth/test_multipart_bypass_repro.py
tests/unit/proxy/auth/test_proxy_routes.py
tests/unit/proxy/middleware/test_request_size_limit_middleware.py
tests/unit/proxy/test_proxy_config_unit_test.py
tests/unit/proxy/test_proxy_token_counter.py
tests/unit/proxy/test_server_root_path.py
workers: 4
dist: loadscope
timeout: 15
- test-group: mcp-oauth
test-path: >-
tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py
tests/unit/proxy/_experimental/mcp_server/outbound_credentials
workers: 4
dist: loadscope
timeout: 15
# ---- logging: split into 2 shards ----
- test-group: custom-logging
test-path: >-
tests/proxy_unit_tests/test_proxy_custom_logger.py
tests/unit/proxy/test_custom_callback_input.py
tests/unit/proxy/test_custom_logger_s3_gcs.py
workers: 4
dist: loadscope
timeout: 15
- test-group: logging-misc
test-path: >-
tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
tests/unit/proxy/spend_tracking/test_search_api_logging.py
tests/unit/proxy/test_proxy_reject_logging.py
workers: 4
dist: loadscope
timeout: 15
- test-group: db-and-spend
test-path: >-
tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
tests/unit/proxy/db/test_update_daily_tag_spend.py
tests/unit/proxy/test_db_schema_changes.py
tests/unit/proxy/test_prisma_client_backoff_retry.py
tests/unit/proxy/test_update_spend.py
tests/unit/skills/test_skills_db.py
workers: 4
dist: loadscope
timeout: 15
# ---- guardrails + budget + hooks: split into 2 ----
- test-group: guardrails-hooks
test-path: >-
tests/unit/proxy/hooks/test_banned_keyword_list.py
tests/unit/proxy/test_proxy_setting_guardrails.py
tests/unit/proxy/test_unit_test_proxy_hooks.py
workers: 4
dist: loadscope
timeout: 15
- test-group: budgets
test-path: >-
tests/unit/proxy/auth/test_default_end_user_budget_simple.py
tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
tests/unit/proxy/test_zero_cost_model_budget_bypass.py
workers: 4
dist: loadscope
timeout: 15
- test-group: endpoints-and-responses
test-path: >-
tests/proxy_unit_tests/test_proxy_exception_mapping.py
tests/unit/proxy/lens
tests/unit/proxy/auth/test_models_fallback_endpoint.py
tests/unit/proxy/common_utils/test_check_batch_cost.py
tests/unit/proxy/common_utils/test_check_responses_cost.py
tests/unit/proxy/common_utils/test_realtime_cache.py
tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
tests/unit/proxy/response_polling
tests/unit/proxy/test_custom_tokenizer_bug.py
tests/unit/proxy/test_get_favicon.py
tests/unit/proxy/test_get_image.py
tests/unit/proxy/test_prompt_test_endpoint.py
tests/unit/proxy/test_reducto_ocr_route.py
tests/unit/proxy/test_response_polling_pre_call_checks.py
tests/unit/proxy/test_ui_path_detection.py
workers: 4
dist: loadscope
timeout: 15
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: ${{ matrix.test-path }}
workers: ${{ matrix.workers }}
reruns: 2
timeout-minutes: ${{ matrix.timeout }}
dist: ${{ matrix.dist }}
artifact-name: proxy-db-${{ matrix.test-group }}

213
.github/workflows/test-unit.yml vendored
View file

@ -28,11 +28,6 @@ concurrency:
# defaults. An absent matrix key renders as an empty string, which is not a
# number, so a partially-specified entry would fail the call rather than fall
# back to the default.
#
# tests/unit/proxy keeps its own caller (test-unit-proxy-db.yml): it is already
# a matrix and carries a shard-coverage guard that reads that file by name.
# Folding it in here is a follow-up, together with generalising that guard into
# assert_ci_coverage.py.
jobs:
rust-bridge:
name: Build the Rust bridge
@ -508,3 +503,211 @@ jobs:
job-timeout-minutes: ${{ matrix.job-timeout-minutes }}
dist: ${{ matrix.dist || 'loadscope' }}
artifact-name: ${{ matrix.artifact-name }}
lens-python-310:
name: Lens Python 3.10
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
python-version: "3.10"
test-path: tests/unit/proxy/lens/test_inference.py
workers: 0
reruns: 0
timeout-minutes: 5
artifact-name: lens-python-310
# Fast guard — fails the workflow when a test directory or file inside a sharded
# tree is claimed by no shard. The semantic-shard design has no catch-all bucket,
# so an unassigned child runs nowhere; assert_ci_coverage.py holds the tree list
# and reads the same test-path keys the coverage census does.
assert-shard-coverage:
runs-on: ubuntu-latest
timeout-minutes: 2
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Assert every test directory and file is claimed by a shard
run: python3 .github/scripts/assert_ci_coverage.py --shards
proxy-db:
needs: assert-shard-coverage
# Display only the semantic shard name in the checks UI instead of GHA's
# default "proxy-db (key-generation, tests/unit/proxy/…, 0, loadscope, 20)"
# which includes every matrix field and gets truncated past the test-path.
name: ${{ matrix.test-group }}
permissions:
contents: read
id-token: write
pull-requests: write
strategy:
fail-fast: false
matrix:
include:
# Must run serially — event-loop conflict with the logging worker.
- test-group: key-generation
test-path: >-
tests/unit/proxy/management_endpoints/test_key_generate_prisma.py
workers: 0
dist: loadscope
timeout: 20
# ---- auth: split into 2 shards ----
- test-group: auth-checks
test-path: >-
tests/unit/proxy/auth/test_auth_checks.py
tests/unit/proxy/auth/test_user_api_key_auth.py
tests/unit/proxy/test_credential_slot_registry.py
tests/unit/proxy/test_deprecated_key_grace_period.py
workers: 4
dist: loadscope
timeout: 15
- test-group: jwt-and-keys
test-path: >-
tests/unit/proxy/auth/test_jwt.py
tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
tests/unit/proxy/test_proxy_custom_auth.py
workers: 4
dist: loadscope
timeout: 15
# ---- test_proxy_utils.py, single shard, worksteal distribution ----
- test-group: proxy-utils
test-path: >-
tests/unit/proxy/test_proxy_utils.py
workers: 4
dist: worksteal
timeout: 15
# ---- proxy server: split into 2 shards ----
- test-group: proxy-server-core
test-path: >-
tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py
tests/unit/proxy/test__lazy_features.py
tests/unit/proxy/test_aproxy_startup.py
tests/unit/proxy/test_proxy_server.py
workers: 4
dist: loadscope
timeout: 15
- test-group: proxy-runtime
test-path: >-
tests/unit/proxy/auth/test_multipart_bypass_repro.py
tests/unit/proxy/auth/test_proxy_routes.py
tests/unit/proxy/middleware/test_request_size_limit_middleware.py
tests/unit/proxy/test_proxy_config_unit_test.py
tests/unit/proxy/test_proxy_token_counter.py
tests/unit/proxy/test_server_root_path.py
workers: 4
dist: loadscope
timeout: 15
- test-group: mcp-oauth
test-path: >-
tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py
tests/unit/proxy/_experimental/mcp_server/outbound_credentials
workers: 4
dist: loadscope
timeout: 15
# ---- logging: split into 2 shards ----
- test-group: custom-logging
test-path: >-
tests/proxy_unit_tests/test_proxy_custom_logger.py
tests/unit/proxy/test_custom_callback_input.py
tests/unit/proxy/test_custom_logger_s3_gcs.py
workers: 4
dist: loadscope
timeout: 15
- test-group: logging-misc
test-path: >-
tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
tests/unit/proxy/spend_tracking/test_search_api_logging.py
tests/unit/proxy/test_proxy_reject_logging.py
workers: 4
dist: loadscope
timeout: 15
- test-group: db-and-spend
test-path: >-
tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
tests/unit/proxy/db/test_update_daily_tag_spend.py
tests/unit/proxy/test_db_schema_changes.py
tests/unit/proxy/test_prisma_client_backoff_retry.py
tests/unit/proxy/test_update_spend.py
tests/unit/skills/test_skills_db.py
workers: 4
dist: loadscope
timeout: 15
# ---- guardrails + budget + hooks: split into 2 ----
- test-group: guardrails-hooks
test-path: >-
tests/unit/proxy/hooks/test_banned_keyword_list.py
tests/unit/proxy/test_proxy_setting_guardrails.py
tests/unit/proxy/test_unit_test_proxy_hooks.py
workers: 4
dist: loadscope
timeout: 15
- test-group: budgets
test-path: >-
tests/unit/proxy/auth/test_default_end_user_budget_simple.py
tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
tests/unit/proxy/test_zero_cost_model_budget_bypass.py
workers: 4
dist: loadscope
timeout: 15
- test-group: endpoints-and-responses
test-path: >-
tests/proxy_unit_tests/test_proxy_exception_mapping.py
tests/unit/proxy/lens
tests/unit/proxy/auth/test_models_fallback_endpoint.py
tests/unit/proxy/common_utils/test_check_batch_cost.py
tests/unit/proxy/common_utils/test_check_responses_cost.py
tests/unit/proxy/common_utils/test_realtime_cache.py
tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
tests/unit/proxy/response_polling
tests/unit/proxy/test_custom_tokenizer_bug.py
tests/unit/proxy/test_get_favicon.py
tests/unit/proxy/test_get_image.py
tests/unit/proxy/test_prompt_test_endpoint.py
tests/unit/proxy/test_reducto_ocr_route.py
tests/unit/proxy/test_response_polling_pre_call_checks.py
tests/unit/proxy/test_ui_path_detection.py
workers: 4
dist: loadscope
timeout: 15
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: ${{ matrix.test-path }}
workers: ${{ matrix.workers }}
reruns: 2
timeout-minutes: ${{ matrix.timeout }}
dist: ${{ matrix.dist }}
artifact-name: proxy-db-${{ matrix.test-group }}
unit-passed:
name: unit passed
needs: [rust-bridge, unit, lens-python-310, assert-shard-coverage, proxy-db]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 2
permissions: {}
steps:
- name: Require every unit job to succeed
env:
NEEDS: ${{ toJSON(needs) }}
run: |
jq -r 'to_entries[] | "\(.key): \(.value.result)"' <<< "$NEEDS"
jq -e 'all(.[]; .result == "success")' <<< "$NEEDS" > /dev/null

68
tests/code_coverage_tests/test_unit_passed_gate.py Normal file
View file

@ -0,0 +1,68 @@
import json
import os
import subprocess
from pathlib import Path
from typing import Final
import pytest
import yaml
_UNIT_WORKFLOW: Final = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "test-unit.yml"
_GATE_JOB: Final = "unit-passed"
def _jobs() -> dict[str, dict[str, object]]:
return yaml.safe_load(_UNIT_WORKFLOW.read_text())["jobs"]
def _gate_script() -> str:
steps: Final = _jobs()[_GATE_JOB]["steps"]
assert isinstance(steps, list) and len(steps) == 1
return steps[0]["run"]
def _run_gate(results: dict[str, str]) -> subprocess.CompletedProcess[str]:
needs: Final = {job: {"result": result, "outputs": {}} for job, result in results.items()}
return subprocess.run(
("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", _gate_script()),
env={**os.environ, "NEEDS": json.dumps(needs)},
capture_output=True,
text=True,
timeout=30,
check=False,
)
def _needed_jobs() -> tuple[str, ...]:
needs: Final = _jobs()[_GATE_JOB]["needs"]
assert isinstance(needs, list)
return tuple(needs)
def test_the_gate_passes_when_every_needed_job_succeeded() -> None:
jobs: Final = _needed_jobs()
assert jobs
result: Final = _run_gate(dict.fromkeys(jobs, "success"))
assert result.returncode == 0, result.stdout + result.stderr
assert all(f"{job}: success" in result.stdout for job in jobs), result.stdout
@pytest.mark.parametrize("outcome", ("failure", "cancelled", "skipped"))
def test_the_gate_fails_when_any_needed_job_did_not_succeed(outcome: str) -> None:
jobs: Final = _needed_jobs()
assert len(jobs) > 1
result: Final = _run_gate({**dict.fromkeys(jobs, "success"), jobs[-1]: outcome})
assert result.returncode != 0
assert f"{jobs[-1]}: {outcome}" in result.stdout, result.stdout
def test_the_gate_waits_for_every_other_job_and_reports_even_when_they_fail() -> None:
jobs: Final = _jobs()
gate: Final = jobs[_GATE_JOB]
assert set(_needed_jobs()) == set(jobs) - {_GATE_JOB}
assert gate["if"] == "always()"