From 489d587332a1577436fe620867d55080541b88cd Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Sat, 24 Jan 2026 09:22:58 -0800 Subject: [PATCH] CI/CD fixes - split local testing --- .circleci/config.yml | 160 ++++++++++++++++-- ...odel_prices_and_context_window_backup.json | 13 +- 2 files changed, 159 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 887c2a6f13a..77f3c2b9a7d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,7 +112,7 @@ jobs: python -m mypy . cd .. no_output_timeout: 10m - local_testing: + local_testing_part1: docker: - image: cimg/python:3.12 auth: @@ -205,12 +205,12 @@ jobs: # Run pytest and generate JUnit XML report - run: - name: Run tests + name: Run tests (Part 1 - A-M) command: | mkdir test-results - # Discover test files - TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") + # Discover test files (A-M) + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py") echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ @@ -229,8 +229,8 @@ jobs: - run: name: Rename the coverage files command: | - mv coverage.xml local_testing_coverage.xml - mv .coverage local_testing_coverage + mv coverage.xml local_testing_part1_coverage.xml + mv .coverage local_testing_part1_coverage # Store test results - store_test_results: @@ -238,8 +238,136 @@ jobs: - persist_to_workspace: root: . paths: - - local_testing_coverage.xml - - local_testing_coverage + - local_testing_part1_coverage.xml + - local_testing_part1_coverage + local_testing_part2: + docker: + - image: cimg/python:3.12 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + parallelism: 4 + steps: + - checkout + - setup_google_dns + - run: + name: Show git commit hash + command: | + echo "Git commit hash: $CIRCLE_SHA1" + + - restore_cache: + keys: + - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r .circleci/requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install "pytest-cov==5.0.0" + pip install "mypy==1.18.2" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install pyarrow + pip install "boto3==1.36.0" + pip install "aioboto3==13.4.0" + pip install langchain + pip install lunary==0.2.5 + pip install "azure-identity==1.16.1" + pip install "langfuse==2.59.7" + pip install "logfire==0.29.0" + pip install numpydoc + pip install traceloop-sdk==0.21.1 + pip install opentelemetry-api==1.25.0 + pip install opentelemetry-sdk==1.25.0 + pip install opentelemetry-exporter-otlp==1.25.0 + pip install openai==1.100.1 + pip install prisma==0.11.0 + pip install "detect_secrets==1.5.0" + pip install "httpx==0.24.1" + pip install "respx==0.22.0" + pip install fastapi + pip install "gunicorn==21.2.0" + pip install "anyio==4.2.0" + pip install "aiodynamo==23.10.1" + pip install "asyncio==3.4.3" + pip install "apscheduler==3.10.4" + pip install "PyGithub==1.59.1" + pip install argon2-cffi + pip install "pytest-mock==3.12.0" + pip install python-multipart + pip install google-cloud-aiplatform + pip install prometheus-client==0.20.0 + pip install "pydantic==2.10.2" + pip install "diskcache==5.6.1" + pip install "Pillow==10.3.0" + pip install "jsonschema==4.22.0" + pip install "pytest-xdist==3.6.1" + pip install "pytest-timeout==2.2.0" + pip install "websockets==13.1.0" + pip install semantic_router --no-deps + pip install aurelio_sdk --no-deps + pip uninstall posthog -y + - setup_litellm_enterprise_pip + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Run prisma ./docker/entrypoint.sh + command: | + set +e + chmod +x docker/entrypoint.sh + ./docker/entrypoint.sh + set -e + - run: + name: Black Formatting + command: | + cd litellm + python -m pip install black + python -m black . + cd .. + + # Run pytest and generate JUnit XML report + - run: + name: Run tests (Part 2 - N-Z) + command: | + mkdir test-results + + # Discover test files (N-Z) + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py") + + echo "$TEST_FILES" | circleci tests run \ + --split-by=timings \ + --verbose \ + --command="xargs python -m pytest \ + -vv \ + --cov=litellm \ + --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=20 \ + -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ + -n 4 \ + --timeout=300 \ + --timeout_method=thread" + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml local_testing_part2_coverage.xml + mv .coverage local_testing_part2_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - local_testing_part2_coverage.xml + - local_testing_part2_coverage langfuse_logging_unit_tests: docker: - image: cimg/python:3.11 @@ -3300,7 +3428,7 @@ jobs: python -m venv venv . venv/bin/activate pip install coverage - coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage + coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage coverage xml - codecov/upload: file: ./coverage.xml @@ -3755,7 +3883,13 @@ workflows: only: - main - /litellm_.*/ - - local_testing: + - local_testing_part1: + filters: + branches: + only: + - main + - /litellm_.*/ + - local_testing_part2: filters: branches: only: @@ -4060,7 +4194,8 @@ workflows: - litellm_proxy_unit_testing_part2 - litellm_security_tests - langfuse_logging_unit_tests - - local_testing + - local_testing_part1 + - local_testing_part2 - litellm_assistants_api_testing - auth_ui_unit_tests - db_migration_disable_update_check: @@ -4103,7 +4238,8 @@ workflows: - publish_to_pypi: requires: - mypy_linting - - local_testing + - local_testing_part1 + - local_testing_part2 - build_and_test - e2e_openai_endpoints - test_bad_database_url diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 209c0794e50..d958ea4503a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9788,6 +9788,7 @@ "supports_tool_choice": true }, "deepinfra/google/gemini-2.0-flash-001": { + "deprecation_date": "2026-03-31", "max_tokens": 1000000, "max_input_tokens": 1000000, "max_output_tokens": 1000000, @@ -12105,6 +12106,7 @@ }, "gemini-2.0-flash": { "cache_read_input_token_cost": 2.5e-08, + "deprecation_date": "2026-03-31", "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-language-models", @@ -12144,7 +12146,7 @@ }, "gemini-2.0-flash-001": { "cache_read_input_token_cost": 3.75e-08, - "deprecation_date": "2026-02-05", + "deprecation_date": "2026-03-31", "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1.5e-07, "litellm_provider": "vertex_ai-language-models", @@ -12230,6 +12232,7 @@ }, "gemini-2.0-flash-lite": { "cache_read_input_token_cost": 1.875e-08, + "deprecation_date": "2026-03-31", "input_cost_per_audio_token": 7.5e-08, "input_cost_per_token": 7.5e-08, "litellm_provider": "vertex_ai-language-models", @@ -12265,7 +12268,7 @@ }, "gemini-2.0-flash-lite-001": { "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-02-25", + "deprecation_date": "2026-03-31", "input_cost_per_audio_token": 7.5e-08, "input_cost_per_token": 7.5e-08, "litellm_provider": "vertex_ai-language-models", @@ -13930,6 +13933,7 @@ }, "gemini/gemini-2.0-flash": { "cache_read_input_token_cost": 2.5e-08, + "deprecation_date": "2026-03-31", "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1e-07, "litellm_provider": "gemini", @@ -13970,6 +13974,7 @@ }, "gemini/gemini-2.0-flash-001": { "cache_read_input_token_cost": 2.5e-08, + "deprecation_date": "2026-03-31", "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1e-07, "litellm_provider": "gemini", @@ -14057,6 +14062,7 @@ }, "gemini/gemini-2.0-flash-lite": { "cache_read_input_token_cost": 1.875e-08, + "deprecation_date": "2026-03-31", "input_cost_per_audio_token": 7.5e-08, "input_cost_per_token": 7.5e-08, "litellm_provider": "gemini", @@ -23158,6 +23164,7 @@ "supports_tool_choice": true }, "openrouter/google/gemini-2.0-flash-001": { + "deprecation_date": "2026-03-31", "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1e-07, "litellm_provider": "openrouter", @@ -27695,6 +27702,7 @@ "output_cost_per_token": 9e-07 }, "vercel_ai_gateway/google/gemini-2.0-flash": { + "deprecation_date": "2026-03-31", "input_cost_per_token": 1.5e-07, "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1048576, @@ -27704,6 +27712,7 @@ "output_cost_per_token": 6e-07 }, "vercel_ai_gateway/google/gemini-2.0-flash-lite": { + "deprecation_date": "2026-03-31", "input_cost_per_token": 7.5e-08, "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1048576,