mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
d7e4b1bdd0
tests/proxy_unit_tests had a 30-line YAML parser inlined in its workflow that failed the run when a test file there belonged to no shard. tests/test_litellm is sharded the same way, with no catch-all bucket, and had no such guard: a new directory under it (or under its proxy subtree) is collected by nothing and runs nowhere, and the coverage census cannot see it because a token like tests/test_litellm/test_*.py already answers 'yes, that tree runs'. The two questions differ. The census asks whether a file runs at all, so an ancestor path standing in for everything beneath it is a fine answer. Shard assignment asks which shard owns a child, and there that same ancestor path is precisely the bug. _token_covers keeps the first meaning; _token_names adds the second, and the guard now walks a list of sharded trees rather than one hardcoded directory. Both read the same test-path keys, so there is one workflow parser. A directory needs a shard when it holds a test file, not when it is named test_*. That drops the hardcoded test_configs exception and keeps fixture directories like expected_fine_tuning_api out on their own merits. The job keeps its name and its workflow, since assert-shard-coverage is a required status check on litellm_internal_staging. Verified red-first: a planted directory under tests/test_litellm, a planted directory under tests/test_litellm/proxy, and a planted file under tests/proxy_unit_tests each fail the guard, while a fixture-only directory does not. 327 children across the three trees are assigned today.
212 lines
9 KiB
YAML
212 lines
9 KiB
YAML
name: "Unit Tests: Proxy DB Operations"
|
|
|
|
on:
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
- litellm_internal_staging
|
|
- litellm_oss_staging
|
|
- "litellm_**"
|
|
push:
|
|
branches:
|
|
- main
|
|
- litellm_internal_staging
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
|
|
# Semantic matrix: each shard groups tests by concern (auth, server, logging, …)
|
|
# rather than alphabetical letter ranges. Adding a new test file means adding it
|
|
# to whichever group it belongs to, not reshuffling slices.
|
|
#
|
|
# Design targets:
|
|
# * Every shard runs in <= 7 minutes of wall-clock on the default runner.
|
|
# Most of a shard's time is pytest plugin load + xdist worker imports +
|
|
# pytest-cov instrumentation, not the tests themselves. Keeping per-shard
|
|
# work low and matching worker count to runner cores is what controls it.
|
|
# * `timeout` bounds the pytest step only. Checkout, dependency install, and
|
|
# Prisma client generation draw on a separate allowance in the base
|
|
# workflow, so slow setup shows up as a slow job rather than as a
|
|
# cancelled shard whose tests were passing.
|
|
# * workers: 4 matches the 4-core ubuntu-latest runner. -n 8 on 4 cores
|
|
# oversubscribes 2x and workers fight for CPU during their cold-start
|
|
# imports (measured ~441% CPU for -n 8 locally, i.e. ~55% effective).
|
|
# * test_key_generate_prisma.py stays serial (workers=0) — it has event-loop
|
|
# conflicts with the logging worker when run in parallel.
|
|
# * test_proxy_utils.py runs as a single shard with --dist=worksteal so
|
|
# xdist balances its 188 parametrized cases across workers instead of
|
|
# pinning the whole file to one worker (the default --dist=loadscope
|
|
# behavior for single-file targets).
|
|
jobs:
|
|
# Fast guard — fails the workflow when a test directory or file inside a sharded
|
|
# tree is claimed by no shard. The semantic-shard design has no catch-all bucket,
|
|
# so an unassigned child runs nowhere; assert_ci_coverage.py holds the tree list
|
|
# and reads the same test-path keys the coverage census does.
|
|
assert-shard-coverage:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 2
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
- name: Assert every test directory and file is claimed by a shard
|
|
run: python3 .github/scripts/assert_ci_coverage.py --shards
|
|
|
|
proxy-db:
|
|
needs: assert-shard-coverage
|
|
# Display only the semantic shard name in the checks UI instead of GHA's
|
|
# default "proxy-db (key-generation, tests/proxy_unit_tests/…, 0, loadscope, 20)"
|
|
# which includes every matrix field and gets truncated past the test-path.
|
|
name: ${{ matrix.test-group }}
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
pull-requests: write
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
# Must run serially — event-loop conflict with the logging worker.
|
|
- test-group: key-generation
|
|
test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py"
|
|
workers: 0
|
|
dist: loadscope
|
|
timeout: 20
|
|
|
|
# ---- auth: split into 2 shards ----
|
|
- test-group: auth-checks
|
|
test-path: >-
|
|
tests/proxy_unit_tests/test_auth_checks.py
|
|
tests/proxy_unit_tests/test_user_api_key_auth.py
|
|
tests/proxy_unit_tests/test_deprecated_key_grace_period.py
|
|
workers: 4
|
|
dist: loadscope
|
|
timeout: 15
|
|
- test-group: jwt-and-keys
|
|
test-path: >-
|
|
tests/proxy_unit_tests/test_jwt.py
|
|
tests/proxy_unit_tests/test_jwt_key_mapping.py
|
|
tests/proxy_unit_tests/test_proxy_custom_auth.py
|
|
tests/proxy_unit_tests/test_key_generate_dynamodb.py
|
|
tests/proxy_unit_tests/test_deployed_proxy_keygen.py
|
|
workers: 4
|
|
dist: loadscope
|
|
timeout: 15
|
|
|
|
# ---- test_proxy_utils.py, single shard, worksteal distribution ----
|
|
- test-group: proxy-utils
|
|
test-path: "tests/proxy_unit_tests/test_proxy_utils.py"
|
|
workers: 4
|
|
dist: worksteal
|
|
timeout: 15
|
|
|
|
# ---- proxy server: split into 2 shards ----
|
|
- test-group: proxy-server-core
|
|
test-path: >-
|
|
tests/proxy_unit_tests/test_proxy_server.py
|
|
tests/proxy_unit_tests/test_proxy_server_keys.py
|
|
tests/proxy_unit_tests/test_proxy_server_spend.py
|
|
tests/proxy_unit_tests/test_aproxy_startup.py
|
|
workers: 4
|
|
dist: loadscope
|
|
timeout: 15
|
|
- test-group: proxy-runtime
|
|
test-path: >-
|
|
tests/proxy_unit_tests/test_proxy_config_unit_test.py
|
|
tests/proxy_unit_tests/test_proxy_routes.py
|
|
tests/proxy_unit_tests/test_proxy_gunicorn.py
|
|
tests/proxy_unit_tests/test_server_root_path.py
|
|
tests/proxy_unit_tests/test_proxy_pass_user_config.py
|
|
tests/proxy_unit_tests/test_proxy_token_counter.py
|
|
tests/proxy_unit_tests/test_request_size_limit_middleware.py
|
|
tests/proxy_unit_tests/test_multipart_bypass_repro.py
|
|
workers: 4
|
|
dist: loadscope
|
|
timeout: 15
|
|
|
|
# ---- logging: split into 2 shards ----
|
|
- test-group: custom-logging
|
|
test-path: >-
|
|
tests/proxy_unit_tests/test_custom_callback_input.py
|
|
tests/proxy_unit_tests/test_custom_logger_s3_gcs.py
|
|
tests/proxy_unit_tests/test_proxy_custom_logger.py
|
|
workers: 4
|
|
dist: loadscope
|
|
timeout: 15
|
|
- test-group: logging-misc
|
|
test-path: >-
|
|
tests/proxy_unit_tests/test_proxy_reject_logging.py
|
|
tests/proxy_unit_tests/test_audit_logs_proxy.py
|
|
tests/proxy_unit_tests/test_search_api_logging.py
|
|
workers: 4
|
|
dist: loadscope
|
|
timeout: 15
|
|
|
|
- test-group: db-and-spend
|
|
test-path: >-
|
|
tests/proxy_unit_tests/test_prisma_client_backoff_retry.py
|
|
tests/proxy_unit_tests/test_db_schema_changes.py
|
|
tests/proxy_unit_tests/test_e2e_pod_lock_manager.py
|
|
tests/proxy_unit_tests/test_skills_db.py
|
|
tests/proxy_unit_tests/test_update_daily_tag_spend.py
|
|
tests/proxy_unit_tests/test_update_spend.py
|
|
tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py
|
|
workers: 4
|
|
dist: loadscope
|
|
timeout: 15
|
|
|
|
# ---- guardrails + budget + hooks: split into 2 ----
|
|
- test-group: guardrails-hooks
|
|
test-path: >-
|
|
tests/proxy_unit_tests/test_proxy_setting_guardrails.py
|
|
tests/proxy_unit_tests/test_banned_keyword_list.py
|
|
tests/proxy_unit_tests/test_unit_test_proxy_hooks.py
|
|
workers: 4
|
|
dist: loadscope
|
|
timeout: 15
|
|
- test-group: budgets
|
|
test-path: >-
|
|
tests/proxy_unit_tests/test_default_end_user_budget_simple.py
|
|
tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py
|
|
tests/proxy_unit_tests/test_zero_cost_model_budget_bypass.py
|
|
workers: 4
|
|
dist: loadscope
|
|
timeout: 15
|
|
|
|
- test-group: endpoints-and-responses
|
|
test-path: >-
|
|
tests/proxy_unit_tests/test_blog_posts_endpoint.py
|
|
tests/proxy_unit_tests/test_models_fallback_endpoint.py
|
|
tests/proxy_unit_tests/test_google_endpoint_routing.py
|
|
tests/proxy_unit_tests/test_google_gemini_proxy_request.py
|
|
tests/proxy_unit_tests/test_gemini_agents_endpoints.py
|
|
tests/proxy_unit_tests/test_get_favicon.py
|
|
tests/proxy_unit_tests/test_get_image.py
|
|
tests/proxy_unit_tests/test_reducto_ocr_route.py
|
|
tests/proxy_unit_tests/test_ui_path_detection.py
|
|
tests/proxy_unit_tests/test_prompt_test_endpoint.py
|
|
tests/proxy_unit_tests/test_check_batch_cost.py
|
|
tests/proxy_unit_tests/test_check_responses_cost.py
|
|
tests/proxy_unit_tests/test_response_polling_handler.py
|
|
tests/proxy_unit_tests/test_response_polling_pre_call_checks.py
|
|
tests/proxy_unit_tests/test_realtime_cache.py
|
|
tests/proxy_unit_tests/test_proxy_exception_mapping.py
|
|
tests/proxy_unit_tests/test_custom_tokenizer_bug.py
|
|
tests/proxy_unit_tests/test_model_response_typing
|
|
workers: 4
|
|
dist: loadscope
|
|
timeout: 15
|
|
uses: ./.github/workflows/_test-unit-base.yml
|
|
with:
|
|
test-path: ${{ matrix.test-path }}
|
|
workers: ${{ matrix.workers }}
|
|
reruns: 2
|
|
timeout-minutes: ${{ matrix.timeout }}
|
|
dist: ${{ matrix.dist }}
|
|
artifact-name: proxy-db-${{ matrix.test-group }}
|