From 68deda3da754430fcae3f96a810862ae5361ddfc Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 20 Aug 2026 10:07:02 -0700 Subject: [PATCH] refactor(ci): fold the nine thin unit-shard callers into one matrix (#37590) Nine workflow files existed only to make a single call to _test-unit-base.yml with a different test-path. Adding a shard meant adding a file; changing anything shared meant editing nine. One matrix caller replaces them, so a shard is now one entry. Check names are unchanged, which is the whole constraint. A reusable-workflow job reports as " / ", so setting `name` to the shard id alone reproduces today's context strings exactly: the eleven the matrix produces are eleven of the twenty-three "/ Run tests" contexts the branch ruleset requires, matched string for string. No ruleset edit is needed and none should be made for this. Every matrix entry states its timeouts even where they equal the base defaults. An absent matrix key renders as an empty string rather than falling back, and an empty string is not a number, so a partially-specified entry would fail the call. tests/proxy_unit_tests keeps test-unit-proxy-db.yml. It is already a matrix and its shard-coverage guard reads that file by name, so folding it in belongs with generalising that guard into assert_ci_coverage.py rather than here. Its twelve shards are the remaining required contexts. test-unit-documentation.yml stays too: it does not call the base workflow. --- .github/workflows/test-unit-core-utils.yml | 31 --- .../test-unit-enterprise-routing.yml | 35 --- .github/workflows/test-unit-integrations.yml | 31 --- .github/workflows/test-unit-llm-providers.yml | 47 ---- .github/workflows/test-unit-misc.yml | 53 ----- .github/workflows/test-unit-proxy-auth.yml | 31 --- .../workflows/test-unit-proxy-endpoints.yml | 81 ------- .github/workflows/test-unit-proxy-infra.yml | 42 ---- .../test-unit-responses-caching-types.yml | 31 --- .github/workflows/test-unit.yml | 219 ++++++++++++++++++ 10 files changed, 219 insertions(+), 382 deletions(-) delete mode 100644 .github/workflows/test-unit-core-utils.yml delete mode 100644 .github/workflows/test-unit-enterprise-routing.yml delete mode 100644 .github/workflows/test-unit-integrations.yml delete mode 100644 .github/workflows/test-unit-llm-providers.yml delete mode 100644 .github/workflows/test-unit-misc.yml delete mode 100644 .github/workflows/test-unit-proxy-auth.yml delete mode 100644 .github/workflows/test-unit-proxy-endpoints.yml delete mode 100644 .github/workflows/test-unit-proxy-infra.yml delete mode 100644 .github/workflows/test-unit-responses-caching-types.yml create mode 100644 .github/workflows/test-unit.yml diff --git a/.github/workflows/test-unit-core-utils.yml b/.github/workflows/test-unit-core-utils.yml deleted file mode 100644 index a01f09559c6..00000000000 --- a/.github/workflows/test-unit-core-utils.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: "Unit Tests: Core Utilities" - -on: - pull_request: - branches: - - main - - litellm_internal_staging - - litellm_oss_staging - - "litellm_**" - push: - branches: - - main - - litellm_internal_staging - -permissions: - contents: read - id-token: write - pull-requests: write - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - core-utils: - uses: ./.github/workflows/_test-unit-base.yml - with: - test-path: "tests/test_litellm/litellm_core_utils" - workers: 2 - reruns: 1 - artifact-name: core-utils diff --git a/.github/workflows/test-unit-enterprise-routing.yml b/.github/workflows/test-unit-enterprise-routing.yml deleted file mode 100644 index a64f00f4744..00000000000 --- a/.github/workflows/test-unit-enterprise-routing.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: "Unit Tests: Enterprise, Google GenAI & Routing" - -on: - pull_request: - branches: - - main - - litellm_internal_staging - - litellm_oss_staging - - "litellm_**" - push: - branches: - - main - - litellm_internal_staging - -permissions: - contents: read - id-token: write - pull-requests: write - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - enterprise-routing: - uses: ./.github/workflows/_test-unit-base.yml - with: - test-path: >- - tests/test_litellm/enterprise - tests/test_litellm/google_genai - tests/test_litellm/router_utils - tests/test_litellm/router_strategy - workers: 2 - reruns: 2 - artifact-name: enterprise-routing diff --git a/.github/workflows/test-unit-integrations.yml b/.github/workflows/test-unit-integrations.yml deleted file mode 100644 index 39752cf8e5d..00000000000 --- a/.github/workflows/test-unit-integrations.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: "Unit Tests: Integrations (Callbacks & Logging)" - -on: - pull_request: - branches: - - main - - litellm_internal_staging - - litellm_oss_staging - - "litellm_**" - push: - branches: - - main - - litellm_internal_staging - -permissions: - contents: read - id-token: write - pull-requests: write - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - integrations: - uses: ./.github/workflows/_test-unit-base.yml - with: - test-path: "tests/test_litellm/integrations" - workers: 2 - reruns: 3 - artifact-name: integrations diff --git a/.github/workflows/test-unit-llm-providers.yml b/.github/workflows/test-unit-llm-providers.yml deleted file mode 100644 index 4d1c921f723..00000000000 --- a/.github/workflows/test-unit-llm-providers.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: "Unit Tests: LLM Provider Transformations" - -on: - pull_request: - branches: - - main - - litellm_internal_staging - - litellm_oss_staging - - "litellm_**" - push: - branches: - - main - - litellm_internal_staging - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - vertex-ai: - name: Vertex AI - permissions: - contents: read - id-token: write - pull-requests: write - uses: ./.github/workflows/_test-unit-base.yml - with: - test-path: "tests/test_litellm/llms/vertex_ai" - workers: 1 - reruns: 2 - artifact-name: llm-vertex-ai - - other-providers: - name: All Other Providers - permissions: - contents: read - id-token: write - pull-requests: write - uses: ./.github/workflows/_test-unit-base.yml - with: - test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" - workers: 2 - reruns: 2 - artifact-name: llm-other-providers diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml deleted file mode 100644 index 123a31e23f7..00000000000 --- a/.github/workflows/test-unit-misc.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: "Unit Tests: MCP, Secrets, Containers & Misc" - -on: - pull_request: - branches: - - main - - litellm_internal_staging - - litellm_oss_staging - - "litellm_**" - push: - branches: - - main - - litellm_internal_staging - -permissions: - contents: read - id-token: write - pull-requests: write - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - misc: - uses: ./.github/workflows/_test-unit-base.yml - with: - test-path: >- - tests/test_litellm/batches - tests/test_litellm/secret_managers - tests/test_litellm/a2a_protocol - tests/test_litellm/anthropic_interface - tests/test_litellm/completion_extras - tests/test_litellm/compression - tests/test_litellm/containers - tests/test_litellm/experimental_mcp_client - tests/test_litellm/models - tests/test_litellm/repositories - tests/test_litellm/images - tests/test_litellm/interactions - tests/test_litellm/ocr - tests/test_litellm/passthrough - tests/test_litellm/rag - tests/test_litellm/realtime_api - tests/test_litellm/rerank_api - tests/test_litellm/sandbox - tests/test_litellm/test_router - tests/test_litellm/vector_stores - tests/test_litellm/videos - tests/test_litellm/test_*.py - workers: 2 - reruns: 2 - artifact-name: misc diff --git a/.github/workflows/test-unit-proxy-auth.yml b/.github/workflows/test-unit-proxy-auth.yml deleted file mode 100644 index c27fe16d611..00000000000 --- a/.github/workflows/test-unit-proxy-auth.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: "Unit Tests: Proxy Auth & Key Management" - -on: - pull_request: - branches: - - main - - litellm_internal_staging - - litellm_oss_staging - - "litellm_**" - push: - branches: - - main - - litellm_internal_staging - -permissions: - contents: read - id-token: write - pull-requests: write - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - proxy-auth: - uses: ./.github/workflows/_test-unit-base.yml - with: - test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client" - workers: 2 - reruns: 2 - artifact-name: proxy-auth diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml deleted file mode 100644 index 3d1d0fcd6c3..00000000000 --- a/.github/workflows/test-unit-proxy-endpoints.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: "Unit Tests: Proxy API Endpoints" - -on: - pull_request: - branches: - - main - - litellm_internal_staging - - litellm_oss_staging - - "litellm_**" - push: - branches: - - main - - litellm_internal_staging - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - proxy-endpoints: - permissions: - contents: read - id-token: write - pull-requests: write - uses: ./.github/workflows/_test-unit-base.yml - with: - test-path: >- - tests/test_litellm/proxy/analytics_endpoints - tests/test_litellm/proxy/management_endpoints - tests/test_litellm/proxy/memory - tests/test_litellm/proxy/guardrails - tests/test_litellm/proxy/management_helpers - tests/test_litellm/proxy/anthropic_endpoints - tests/test_litellm/proxy/google_endpoints - tests/test_litellm/proxy/openai_files_endpoint - tests/test_litellm/proxy/batches_endpoints - tests/test_litellm/proxy/fine_tuning_endpoints - tests/test_litellm/proxy/vector_store_files_endpoints - tests/test_litellm/proxy/video_endpoints - tests/test_litellm/proxy/response_api_endpoints - tests/test_litellm/proxy/image_endpoints - tests/test_litellm/proxy/ocr_endpoints - tests/test_litellm/proxy/vector_store_endpoints - tests/test_litellm/proxy/agent_endpoints - tests/test_litellm/proxy/a2a - tests/test_litellm/proxy/credential_endpoints - tests/test_litellm/proxy/discovery_endpoints - tests/test_litellm/proxy/health_endpoints - tests/test_litellm/proxy/shutdown - tests/test_litellm/proxy/public_endpoints - tests/test_litellm/proxy/prompts - tests/test_litellm/proxy/rag_endpoints - tests/test_litellm/proxy/realtime_endpoints - tests/test_litellm/proxy/ui_crud_endpoints - tests/test_litellm/proxy/config_resolvers - tests/test_litellm/proxy/utils - workers: 2 - reruns: 2 - artifact-name: proxy-endpoints - - # Behavior-pinning tests for litellm/proxy/proxy_server.py. Owns its - # own job (not a path on the proxy-endpoints job above) so its budget - # is independent and its coverage artifact is uploaded separately. - # See: https://www.notion.so/36c43b8acdab81ee845fd5365128a2fc - proxy-server: - permissions: - contents: read - id-token: write - pull-requests: write - uses: ./.github/workflows/_test-unit-base.yml - with: - test-path: tests/test_litellm/proxy/proxy_server - workers: 4 - reruns: 2 - timeout-minutes: 60 - job-timeout-minutes: 95 - artifact-name: proxy-server diff --git a/.github/workflows/test-unit-proxy-infra.yml b/.github/workflows/test-unit-proxy-infra.yml deleted file mode 100644 index 83d95463cdf..00000000000 --- a/.github/workflows/test-unit-proxy-infra.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: "Unit Tests: Proxy Infrastructure" - -on: - pull_request: - branches: - - main - - litellm_internal_staging - - litellm_oss_staging - - "litellm_**" - push: - branches: - - main - - litellm_internal_staging - -permissions: - contents: read - id-token: write - pull-requests: write - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - proxy-infra: - uses: ./.github/workflows/_test-unit-base.yml - with: - test-path: >- - tests/test_litellm/proxy/db - tests/test_litellm/proxy/middleware - tests/test_litellm/proxy/spend_tracking - tests/test_litellm/proxy/pass_through_endpoints - tests/test_litellm/proxy/_experimental - tests/test_litellm/proxy/experimental - tests/test_litellm/proxy/common_utils - tests/test_litellm/proxy/enterprise_billing - tests/test_litellm/proxy/types_utils - tests/test_litellm/proxy/logging_endpoints - tests/test_litellm/proxy/test_*.py - workers: 2 - reruns: 2 - artifact-name: proxy-infra diff --git a/.github/workflows/test-unit-responses-caching-types.yml b/.github/workflows/test-unit-responses-caching-types.yml deleted file mode 100644 index 5b336452069..00000000000 --- a/.github/workflows/test-unit-responses-caching-types.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: "Unit Tests: Responses, Caching & Types" - -on: - pull_request: - branches: - - main - - litellm_internal_staging - - litellm_oss_staging - - "litellm_**" - push: - branches: - - main - - litellm_internal_staging - -permissions: - contents: read - id-token: write - pull-requests: write - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - responses-caching-types: - uses: ./.github/workflows/_test-unit-base.yml - with: - test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types" - workers: 2 - reruns: 2 - artifact-name: responses-caching-types diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml new file mode 100644 index 00000000000..fbba9969c28 --- /dev/null +++ b/.github/workflows/test-unit.yml @@ -0,0 +1,219 @@ +name: "Unit Tests" + +on: + pull_request: + branches: + - main + - litellm_internal_staging + - litellm_oss_staging + - "litellm_**" + push: + branches: + - main + - litellm_internal_staging + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +# One caller for every tests/test_litellm shard, replacing the nine thin workflow +# files that each wrapped a single call to _test-unit-base.yml. Adding a shard is +# now one matrix entry rather than a new file. +# +# `name` is the shard id and nothing else, so each check reports as +# " / Run tests" exactly as it did when the shard had its own file. Those +# strings are the branch ruleset's required contexts, so they are load-bearing: +# renaming an entry renames a required check and the ruleset stops matching it. +# +# Every entry states its timeouts even when they equal the base workflow's +# defaults. An absent matrix key renders as an empty string, which is not a +# number, so a partially-specified entry would fail the call rather than fall +# back to the default. +# +# tests/proxy_unit_tests keeps its own caller (test-unit-proxy-db.yml): it is +# already a matrix and carries a shard-coverage guard that reads that file by +# name. Folding it in here is a follow-up, together with generalising that guard +# into assert_ci_coverage.py. +jobs: + unit: + name: ${{ matrix.shard }} + permissions: + contents: read + id-token: write + pull-requests: write + strategy: + fail-fast: false + matrix: + include: + - shard: core-utils + artifact-name: core-utils + test-path: "tests/test_litellm/litellm_core_utils" + workers: 2 + reruns: 1 + timeout-minutes: 20 + job-timeout-minutes: 55 + + - shard: enterprise-routing + artifact-name: enterprise-routing + test-path: >- + tests/test_litellm/enterprise + tests/test_litellm/google_genai + tests/test_litellm/router_utils + tests/test_litellm/router_strategy + workers: 2 + reruns: 2 + timeout-minutes: 20 + job-timeout-minutes: 55 + + - shard: integrations + artifact-name: integrations + test-path: "tests/test_litellm/integrations" + workers: 2 + reruns: 3 + timeout-minutes: 20 + job-timeout-minutes: 55 + + - shard: Vertex AI + artifact-name: llm-vertex-ai + test-path: "tests/test_litellm/llms/vertex_ai" + workers: 1 + reruns: 2 + timeout-minutes: 20 + job-timeout-minutes: 55 + + - shard: All Other Providers + artifact-name: llm-other-providers + test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" + workers: 2 + reruns: 2 + timeout-minutes: 20 + job-timeout-minutes: 55 + + - shard: misc + artifact-name: misc + test-path: >- + tests/test_litellm/batches + tests/test_litellm/secret_managers + tests/test_litellm/a2a_protocol + tests/test_litellm/anthropic_interface + tests/test_litellm/completion_extras + tests/test_litellm/compression + tests/test_litellm/containers + tests/test_litellm/experimental_mcp_client + tests/test_litellm/models + tests/test_litellm/repositories + tests/test_litellm/images + tests/test_litellm/interactions + tests/test_litellm/ocr + tests/test_litellm/passthrough + tests/test_litellm/rag + tests/test_litellm/realtime_api + tests/test_litellm/rerank_api + tests/test_litellm/sandbox + tests/test_litellm/test_router + tests/test_litellm/vector_stores + tests/test_litellm/videos + tests/test_litellm/test_*.py + workers: 2 + reruns: 2 + timeout-minutes: 20 + job-timeout-minutes: 55 + + - shard: proxy-auth + artifact-name: proxy-auth + test-path: >- + tests/test_litellm/proxy/auth + tests/test_litellm/proxy/hooks + tests/test_litellm/proxy/policy_engine + tests/test_litellm/proxy/client + workers: 2 + reruns: 2 + timeout-minutes: 20 + job-timeout-minutes: 55 + + - shard: proxy-endpoints + artifact-name: proxy-endpoints + test-path: >- + tests/test_litellm/proxy/analytics_endpoints + tests/test_litellm/proxy/management_endpoints + tests/test_litellm/proxy/memory + tests/test_litellm/proxy/guardrails + tests/test_litellm/proxy/management_helpers + tests/test_litellm/proxy/anthropic_endpoints + tests/test_litellm/proxy/google_endpoints + tests/test_litellm/proxy/openai_files_endpoint + tests/test_litellm/proxy/batches_endpoints + tests/test_litellm/proxy/fine_tuning_endpoints + tests/test_litellm/proxy/vector_store_files_endpoints + tests/test_litellm/proxy/video_endpoints + tests/test_litellm/proxy/response_api_endpoints + tests/test_litellm/proxy/image_endpoints + tests/test_litellm/proxy/ocr_endpoints + tests/test_litellm/proxy/vector_store_endpoints + tests/test_litellm/proxy/agent_endpoints + tests/test_litellm/proxy/a2a + tests/test_litellm/proxy/credential_endpoints + tests/test_litellm/proxy/discovery_endpoints + tests/test_litellm/proxy/health_endpoints + tests/test_litellm/proxy/shutdown + tests/test_litellm/proxy/public_endpoints + tests/test_litellm/proxy/prompts + tests/test_litellm/proxy/rag_endpoints + tests/test_litellm/proxy/realtime_endpoints + tests/test_litellm/proxy/ui_crud_endpoints + tests/test_litellm/proxy/config_resolvers + tests/test_litellm/proxy/utils + workers: 2 + reruns: 2 + timeout-minutes: 20 + job-timeout-minutes: 55 + + - shard: proxy-server + artifact-name: proxy-server + test-path: "tests/test_litellm/proxy/proxy_server" + workers: 4 + reruns: 2 + timeout-minutes: 60 + job-timeout-minutes: 95 + + - shard: proxy-infra + artifact-name: proxy-infra + test-path: >- + tests/test_litellm/proxy/db + tests/test_litellm/proxy/middleware + tests/test_litellm/proxy/spend_tracking + tests/test_litellm/proxy/pass_through_endpoints + tests/test_litellm/proxy/_experimental + tests/test_litellm/proxy/experimental + tests/test_litellm/proxy/common_utils + tests/test_litellm/proxy/enterprise_billing + tests/test_litellm/proxy/types_utils + tests/test_litellm/proxy/logging_endpoints + tests/test_litellm/proxy/test_*.py + workers: 2 + reruns: 2 + timeout-minutes: 20 + job-timeout-minutes: 55 + + - shard: responses-caching-types + artifact-name: responses-caching-types + test-path: >- + tests/test_litellm/responses + tests/test_litellm/caching + tests/test_litellm/types + workers: 2 + reruns: 2 + timeout-minutes: 20 + job-timeout-minutes: 55 + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: ${{ matrix.test-path }} + workers: ${{ matrix.workers }} + reruns: ${{ matrix.reruns }} + timeout-minutes: ${{ matrix.timeout-minutes }} + job-timeout-minutes: ${{ matrix.job-timeout-minutes }} + artifact-name: ${{ matrix.artifact-name }}