From 8b54a2a324ec53d4df8883844ef366409c09c43d Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 22 Jan 2026 22:52:33 -0800 Subject: [PATCH] more changes --- .circleci/config.yml | 81 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e51122b72e..d39fcf70d0c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -209,13 +209,13 @@ jobs: command: | pwd ls - mkdir -p test-results + mkdir test-results # Discover test files TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ - --split-by=timings \ + --split-by=filesize \ --verbose \ --command="xargs python -m pytest \ -vv \ @@ -432,9 +432,21 @@ jobs: - run: name: Run tests command: | - pwd - ls - python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "caching or cache" + mkdir test-results + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") + + echo "$TEST_FILES" | circleci tests run \ + --command="xargs python -m pytest \ + tests/local_testing \ + -vv \ + --cov=litellm \ + --cov-report=xml \ + --junitxml=test-results/junit.xml \ + -n 4 \ + --durations=5 \ + -x \ + -k 'caching or cache'" \ + --verbose no_output_timeout: 120m - run: name: Rename the coverage files @@ -535,13 +547,13 @@ jobs: - run: name: Run tests command: | - mkdir -p test-results + mkdir test-results # Find test files only in local_testing TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ - --split-by=timings \ + --split-by=filesize \ --verbose \ --command="xargs python -m pytest \ -k 'router' \ @@ -1119,6 +1131,7 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + parallelism: 4 steps: - checkout @@ -1141,9 +1154,24 @@ jobs: command: | pwd ls - # Add --timeout to kill hanging tests after 120s (2 min) - # Add --durations=20 to show 20 slowest tests for debugging - python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread + + mkdir test-results + + # Collect test files for CircleCI test splitting / rerun failed tests + TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py") + + echo "$TEST_FILES" | circleci tests run \ + --split-by=filesize \ + --verbose \ + --command="xargs python -m pytest \ + -vv \ + --cov=litellm \ + --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=20 \ + -n 4 \ + --timeout=120 \ + --timeout_method=thread" \ no_output_timeout: 120m - run: name: Rename the coverage files @@ -1487,13 +1515,38 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + parallelism: 4 resource_class: xlarge steps: - setup_litellm_test_deps - run: name: Run core tests command: | - python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + pwd + ls + + mkdir test-results + + # Collect core test files for CircleCI rerun/splitting + CORE_TEST_FILES=$(circleci tests glob "tests/test_litellm/**/test_*.py") + + echo "$CORE_TEST_FILES" | circleci tests run \ + --split-by=filesize \ + --verbose \ + --command="xargs python -m pytest \ + -vv \ + --ignore=tests/test_litellm/proxy \ + --ignore=tests/test_litellm/llms \ + --ignore=tests/test_litellm/integrations \ + --ignore=tests/test_litellm/litellm_core_utils \ + --cov=litellm \ + --cov-report=xml \ + --junitxml=test-results/junit-core.xml \ + --durations=10 \ + -n 4 \ + --maxfail=5 \ + --timeout=300 \ + --log-cli-level=WARNING" \ no_output_timeout: 120m - run: name: Rename the coverage files @@ -1827,13 +1880,13 @@ jobs: command: | pwd ls - mkdir -p test-results + mkdir test-results # Discover logging callback test files TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ - --split-by=timings \ + --split-by=filesize \ --verbose \ --command="xargs python -m pytest \ -vv \ @@ -2310,7 +2363,7 @@ jobs: command: | pwd ls - python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests + python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests no_output_timeout: 120m # Store test results