From 7a097ae97f0f085935b5c55560720a1ed1cfe924 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Tue, 17 Feb 2026 15:34:14 -0300 Subject: [PATCH 1/7] fix(ci): reduce parallelism and add retry logic - Reduce workers from 4 to 2 to avoid race conditions - Add --reruns with 2-3 retries per test group - Increase timeout from 15 to 20 minutes - Add better test isolation --- .github/workflows/test-litellm-matrix.yml | 54 ++++++++++++++++++----- poetry.lock | 18 +++++++- pyproject.toml | 1 + 3 files changed, 62 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml index d83fedcb2ae..2f9ddcd27cd 100644 --- a/.github/workflows/test-litellm-matrix.yml +++ b/.github/workflows/test-litellm-matrix.yml @@ -1,4 +1,4 @@ -name: LiteLLM Unit Tests (Matrix) +name: LiteLLM Unit Tests (Matrix) - Improved on: pull_request: @@ -12,7 +12,7 @@ concurrency: jobs: test: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 # Increased from 15 to 20 strategy: fail-fast: false matrix: @@ -20,36 +20,46 @@ jobs: # tests/test_litellm split by subdirectory (~560 files total) - name: "llms" path: "tests/test_litellm/llms" - workers: 4 + workers: 2 # Reduced from 4 to 2 to avoid race conditions + reruns: 2 # Retry flaky tests twice # tests/test_litellm/proxy split by subdirectory (~180 files total) - name: "proxy-guardrails" path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers" - workers: 4 + workers: 2 + reruns: 2 - name: "proxy-core" path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine" - workers: 4 + workers: 2 + reruns: 2 - name: "proxy-misc" path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py" - workers: 4 + workers: 2 + reruns: 2 - name: "integrations" path: "tests/test_litellm/integrations" - workers: 4 + workers: 2 + reruns: 3 # Integration tests tend to be flakier - name: "core-utils" path: "tests/test_litellm/litellm_core_utils" workers: 2 + reruns: 1 - name: "other" path: "tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types" - workers: 4 + workers: 2 + reruns: 2 - name: "root" path: "tests/test_litellm/test_*.py" - workers: 4 + workers: 2 + reruns: 2 # tests/proxy_unit_tests split alphabetically (~48 files total) - name: "proxy-unit-a" path: "tests/proxy_unit_tests/test_[a-o]*.py" workers: 2 + reruns: 1 - name: "proxy-unit-b" path: "tests/proxy_unit_tests/test_[p-z]*.py" workers: 2 + reruns: 1 name: test (${{ matrix.test-group.name }}) @@ -79,6 +89,7 @@ jobs: run: | poetry config virtualenvs.in-project true poetry install --with dev,proxy-dev --extras "proxy semantic-router" + # Note: pytest-rerunfailures is already in pyproject.toml (v14.0 for Python 3.9 compat) poetry run pip install pytest-retry==1.6.3 pytest-xdist google-genai==1.22.0 \ google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core @@ -87,9 +98,32 @@ jobs: cd enterprise && poetry run pip install -e . && cd .. - name: Run tests - ${{ matrix.test-group.name }} + env: + # Prevent tests from trying to use real API keys + LITELLM_CI: "true" + # Set pytest variables for better isolation + PYTHONDONTWRITEBYTECODE: "1" + PYTHONHASHSEED: "0" run: | poetry run pytest ${{ matrix.test-group.path }} \ --tb=short -vv \ --maxfail=10 \ -n ${{ matrix.test-group.workers }} \ - --durations=20 + --reruns ${{ matrix.test-group.reruns }} \ + --reruns-delay 1 \ + --dist loadgroup \ + --durations=20 \ + -m "not no_parallel" + + - name: Run sequential tests (no_parallel) - ${{ matrix.test-group.name }} + if: always() # Run even if parallel tests fail + env: + LITELLM_CI: "true" + PYTHONDONTWRITEBYTECODE: "1" + PYTHONHASHSEED: "0" + run: | + poetry run pytest ${{ matrix.test-group.path }} \ + --tb=short -vv \ + --maxfail=5 \ + -m "no_parallel" \ + --durations=20 || echo "No sequential tests found or tests passed" diff --git a/poetry.lock b/poetry.lock index 82df007de13..e3f3fea1321 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5524,6 +5524,22 @@ psycopg = ">=3.0.0" pytest = ">=6.2" setuptools = "*" +[[package]] +name = "pytest-rerunfailures" +version = "14.0" +description = "pytest plugin to re-run tests to eliminate flaky failures" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pytest-rerunfailures-14.0.tar.gz", hash = "sha256:4a400bcbcd3c7a4ad151ab8afac123d90eca3abe27f98725dc4d9702887d2e92"}, + {file = "pytest_rerunfailures-14.0-py3-none-any.whl", hash = "sha256:4197bdd2eaeffdbf50b5ea6e7236f47ff0e44d1def8dae08e409f536d84e7b32"}, +] + +[package.dependencies] +packaging = ">=17.1" +pytest = ">=7.2" + [[package]] name = "pytest-retry" version = "1.7.0" @@ -7934,4 +7950,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "dfaf1eabfd17db5e30a8dda813872507aa38664fe7681ece2f8fa06ba035d3cf" +content-hash = "d99036fc86de60170dde4a1a9b3f9fdac6eb3610edb6177e70ca86133b71703e" diff --git a/pyproject.toml b/pyproject.toml index 4deb61836b3..307e247ac54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -166,6 +166,7 @@ opentelemetry-exporter-otlp = "^1.28.0" langfuse = "^2.45.0" fastapi-offline = "^1.7.3" fakeredis = "^2.27.1" +pytest-rerunfailures = "^14.0" [tool.poetry.group.proxy-dev.dependencies] prisma = "0.11.0" From e9929d2798ac1cfc81b9a485c540c163f2184488 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Tue, 17 Feb 2026 15:48:09 -0300 Subject: [PATCH 2/7] fix: replace deprecated asyncio.iscoroutinefunction with inspect.iscoroutinefunction - Replace asyncio.iscoroutinefunction() with inspect.iscoroutinefunction() - Add inspect import - Fixes deprecation warning in Python 3.16 The asyncio.iscoroutinefunction is deprecated and will be removed in Python 3.16. Using inspect.iscoroutinefunction is the recommended approach. Co-Authored-By: Claude Sonnet 4.5 --- litellm/litellm_core_utils/logging_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/logging_utils.py b/litellm/litellm_core_utils/logging_utils.py index bf43519afc6..8cde8ccef1c 100644 --- a/litellm/litellm_core_utils/logging_utils.py +++ b/litellm/litellm_core_utils/logging_utils.py @@ -1,5 +1,6 @@ import asyncio import functools +import inspect import time from datetime import datetime from typing import TYPE_CHECKING, Any, List, Optional, Union @@ -270,7 +271,7 @@ def track_llm_api_timing(): verbose_logger.debug(f"Error in service logging: {str(e)}") # Check if the function is async or sync - if asyncio.iscoroutinefunction(func): + if inspect.iscoroutinefunction(func): return async_wrapper return sync_wrapper From bd9b239ed7d28e3cf9c73d1d491339ff7012a32f Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Tue, 17 Feb 2026 15:58:33 -0300 Subject: [PATCH 3/7] fix(ci): address Greptile review feedback - Remove pytest-retry to avoid duplicate retry mechanisms (only use pytest-rerunfailures) - Remove --dist loadgroup flag (no tests use xdist_group marker) - Remove unused LITELLM_CI environment variable - Remove sequential test step with error masking - Simplify workflow for clarity This fixes the issue where tests could be retried 60+ times due to duplicate retry plugins (pytest-retry with retries=20 + pytest-rerunfailures with --reruns 2-3). Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/test-litellm-matrix.yml | 30 ++++------------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml index 2f9ddcd27cd..cdcacb4cd47 100644 --- a/.github/workflows/test-litellm-matrix.yml +++ b/.github/workflows/test-litellm-matrix.yml @@ -1,4 +1,4 @@ -name: LiteLLM Unit Tests (Matrix) - Improved +name: LiteLLM Unit Tests (Matrix) on: pull_request: @@ -89,8 +89,9 @@ jobs: run: | poetry config virtualenvs.in-project true poetry install --with dev,proxy-dev --extras "proxy semantic-router" - # Note: pytest-rerunfailures is already in pyproject.toml (v14.0 for Python 3.9 compat) - poetry run pip install pytest-retry==1.6.3 pytest-xdist google-genai==1.22.0 \ + # pytest-rerunfailures is in pyproject.toml (v14.0 for Python 3.9 compat) + # NOTE: Removed pytest-retry to avoid duplicate retry mechanisms + poetry run pip install google-genai==1.22.0 \ google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core - name: Setup litellm-enterprise @@ -98,12 +99,6 @@ jobs: cd enterprise && poetry run pip install -e . && cd .. - name: Run tests - ${{ matrix.test-group.name }} - env: - # Prevent tests from trying to use real API keys - LITELLM_CI: "true" - # Set pytest variables for better isolation - PYTHONDONTWRITEBYTECODE: "1" - PYTHONHASHSEED: "0" run: | poetry run pytest ${{ matrix.test-group.path }} \ --tb=short -vv \ @@ -111,19 +106,4 @@ jobs: -n ${{ matrix.test-group.workers }} \ --reruns ${{ matrix.test-group.reruns }} \ --reruns-delay 1 \ - --dist loadgroup \ - --durations=20 \ - -m "not no_parallel" - - - name: Run sequential tests (no_parallel) - ${{ matrix.test-group.name }} - if: always() # Run even if parallel tests fail - env: - LITELLM_CI: "true" - PYTHONDONTWRITEBYTECODE: "1" - PYTHONHASHSEED: "0" - run: | - poetry run pytest ${{ matrix.test-group.path }} \ - --tb=short -vv \ - --maxfail=5 \ - -m "no_parallel" \ - --durations=20 || echo "No sequential tests found or tests passed" + --durations=20 From bf157acccce790a268c439cb1f29658c4fc3ce68 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Tue, 17 Feb 2026 16:08:24 -0300 Subject: [PATCH 4/7] fix: restore pytest-xdist to CI workflow - Add pytest-xdist back to pip install line (required for -n flag) - Was accidentally removed when removing pytest-retry - Without pytest-xdist, all CI jobs fail with 'unrecognized option -n' --- .github/workflows/test-litellm-matrix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml index cdcacb4cd47..7aa98036b37 100644 --- a/.github/workflows/test-litellm-matrix.yml +++ b/.github/workflows/test-litellm-matrix.yml @@ -90,8 +90,8 @@ jobs: poetry config virtualenvs.in-project true poetry install --with dev,proxy-dev --extras "proxy semantic-router" # pytest-rerunfailures is in pyproject.toml (v14.0 for Python 3.9 compat) - # NOTE: Removed pytest-retry to avoid duplicate retry mechanisms - poetry run pip install google-genai==1.22.0 \ + # pytest-xdist is required for -n (parallel workers) flag + poetry run pip install pytest-xdist google-genai==1.22.0 \ google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core - name: Setup litellm-enterprise From 02126c5aace20073a282ad9b24566005429aaba9 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Tue, 17 Feb 2026 16:08:40 -0300 Subject: [PATCH 5/7] fix: remove pytest-retry configuration to eliminate duplicate retries - Remove retries=20 and retry_delay=5 from pytest.ini_options - These settings are for pytest-retry plugin (different from pytest-rerunfailures) - Having both pytest-retry + pytest-rerunfailures causes excessive retries - CI workflow now uses only pytest-rerunfailures with --reruns flag --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 307e247ac54..469866c0fa8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -193,8 +193,6 @@ plugins = "pydantic.mypy" [tool.pytest.ini_options] asyncio_mode = "auto" -retries = 20 -retry_delay = 5 markers = [ "asyncio: mark test as an asyncio test", "limit_leaks: mark test with memory limit for leak detection (e.g., '40 MB')", From 48105e650b2978ab82467adf36a244ce1214699e Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Tue, 17 Feb 2026 16:55:37 -0300 Subject: [PATCH 6/7] fix: remove pytest-retry to avoid conflicting retry plugins - Remove pytest-retry from dev dependencies in pyproject.toml - Add pytest-xdist as proper dev dependency (was only in pip install) - Update CI workflow to reflect proper dependency management - Prevents conflict between pytest-retry and pytest-rerunfailures Having both pytest-retry and pytest-rerunfailures installed simultaneously causes unpredictable behavior and excessive retries. --- .github/workflows/test-litellm-matrix.yml | 5 ++--- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml index 7aa98036b37..5590662f5ae 100644 --- a/.github/workflows/test-litellm-matrix.yml +++ b/.github/workflows/test-litellm-matrix.yml @@ -89,9 +89,8 @@ jobs: run: | poetry config virtualenvs.in-project true poetry install --with dev,proxy-dev --extras "proxy semantic-router" - # pytest-rerunfailures is in pyproject.toml (v14.0 for Python 3.9 compat) - # pytest-xdist is required for -n (parallel workers) flag - poetry run pip install pytest-xdist google-genai==1.22.0 \ + # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies + poetry run pip install google-genai==1.22.0 \ google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core - name: Setup litellm-enterprise diff --git a/pyproject.toml b/pyproject.toml index 469866c0fa8..52b1b9452f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,7 +151,7 @@ pytest = "^7.4.3" pytest-mock = "^3.12.0" pytest-asyncio = "^0.21.1" pytest-postgresql = "^6.0.0" -pytest-retry = "^1.6.3" +pytest-xdist = "^3.5.0" requests-mock = "^1.12.1" responses = "^0.25.7" respx = "^0.22.0" From ab912e5c8d3e25e6d623876729a701ebc175d122 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Tue, 17 Feb 2026 17:11:59 -0300 Subject: [PATCH 7/7] refactor: move logging_utils.py deprecation fix to PR #21396 - Remove asyncio.iscoroutinefunction deprecation fix from this PR - This change is better suited for PR #21396 (deprecation warnings) - Keeps PR #21394 focused on CI test reliability improvements --- litellm/litellm_core_utils/logging_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/logging_utils.py b/litellm/litellm_core_utils/logging_utils.py index 8cde8ccef1c..bf43519afc6 100644 --- a/litellm/litellm_core_utils/logging_utils.py +++ b/litellm/litellm_core_utils/logging_utils.py @@ -1,6 +1,5 @@ import asyncio import functools -import inspect import time from datetime import datetime from typing import TYPE_CHECKING, Any, List, Optional, Union @@ -271,7 +270,7 @@ def track_llm_api_timing(): verbose_logger.debug(f"Error in service logging: {str(e)}") # Check if the function is async or sync - if inspect.iscoroutinefunction(func): + if asyncio.iscoroutinefunction(func): return async_wrapper return sync_wrapper