mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
CI/CD fixes - split local testing
This commit is contained in:
Ishaan Jaffer
parent
387cb8fde8
commit
489d587332
2 changed files with 159 additions and 14 deletions
|
|
@ -112,7 +112,7 @@ jobs:
|
|||
python -m mypy .
|
||||
cd ..
|
||||
no_output_timeout: 10m
|
||||
local_testing:
|
||||
local_testing_part1:
|
||||
docker:
|
||||
- image: cimg/python:3.12
|
||||
auth:
|
||||
|
|
@ -205,12 +205,12 @@ jobs:
|
|||
|
||||
# Run pytest and generate JUnit XML report
|
||||
- run:
|
||||
name: Run tests
|
||||
name: Run tests (Part 1 - A-M)
|
||||
command: |
|
||||
mkdir test-results
|
||||
|
||||
# Discover test files
|
||||
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
|
||||
# Discover test files (A-M)
|
||||
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py")
|
||||
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--split-by=timings \
|
||||
|
|
@ -229,8 +229,8 @@ jobs:
|
|||
- run:
|
||||
name: Rename the coverage files
|
||||
command: |
|
||||
mv coverage.xml local_testing_coverage.xml
|
||||
mv .coverage local_testing_coverage
|
||||
mv coverage.xml local_testing_part1_coverage.xml
|
||||
mv .coverage local_testing_part1_coverage
|
||||
|
||||
# Store test results
|
||||
- store_test_results:
|
||||
|
|
@ -238,8 +238,136 @@ jobs:
|
|||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- local_testing_coverage.xml
|
||||
- local_testing_coverage
|
||||
- local_testing_part1_coverage.xml
|
||||
- local_testing_part1_coverage
|
||||
local_testing_part2:
|
||||
docker:
|
||||
- image: cimg/python:3.12
|
||||
auth:
|
||||
username: ${DOCKERHUB_USERNAME}
|
||||
password: ${DOCKERHUB_PASSWORD}
|
||||
working_directory: ~/project
|
||||
parallelism: 4
|
||||
steps:
|
||||
- checkout
|
||||
- setup_google_dns
|
||||
- run:
|
||||
name: Show git commit hash
|
||||
command: |
|
||||
echo "Git commit hash: $CIRCLE_SHA1"
|
||||
|
||||
- restore_cache:
|
||||
keys:
|
||||
- v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
|
||||
- run:
|
||||
name: Install Dependencies
|
||||
command: |
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install -r .circleci/requirements.txt
|
||||
pip install "pytest==7.3.1"
|
||||
pip install "pytest-retry==1.6.3"
|
||||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install "pytest-cov==5.0.0"
|
||||
pip install "mypy==1.18.2"
|
||||
pip install "google-generativeai==0.3.2"
|
||||
pip install "google-cloud-aiplatform==1.43.0"
|
||||
pip install pyarrow
|
||||
pip install "boto3==1.36.0"
|
||||
pip install "aioboto3==13.4.0"
|
||||
pip install langchain
|
||||
pip install lunary==0.2.5
|
||||
pip install "azure-identity==1.16.1"
|
||||
pip install "langfuse==2.59.7"
|
||||
pip install "logfire==0.29.0"
|
||||
pip install numpydoc
|
||||
pip install traceloop-sdk==0.21.1
|
||||
pip install opentelemetry-api==1.25.0
|
||||
pip install opentelemetry-sdk==1.25.0
|
||||
pip install opentelemetry-exporter-otlp==1.25.0
|
||||
pip install openai==1.100.1
|
||||
pip install prisma==0.11.0
|
||||
pip install "detect_secrets==1.5.0"
|
||||
pip install "httpx==0.24.1"
|
||||
pip install "respx==0.22.0"
|
||||
pip install fastapi
|
||||
pip install "gunicorn==21.2.0"
|
||||
pip install "anyio==4.2.0"
|
||||
pip install "aiodynamo==23.10.1"
|
||||
pip install "asyncio==3.4.3"
|
||||
pip install "apscheduler==3.10.4"
|
||||
pip install "PyGithub==1.59.1"
|
||||
pip install argon2-cffi
|
||||
pip install "pytest-mock==3.12.0"
|
||||
pip install python-multipart
|
||||
pip install google-cloud-aiplatform
|
||||
pip install prometheus-client==0.20.0
|
||||
pip install "pydantic==2.10.2"
|
||||
pip install "diskcache==5.6.1"
|
||||
pip install "Pillow==10.3.0"
|
||||
pip install "jsonschema==4.22.0"
|
||||
pip install "pytest-xdist==3.6.1"
|
||||
pip install "pytest-timeout==2.2.0"
|
||||
pip install "websockets==13.1.0"
|
||||
pip install semantic_router --no-deps
|
||||
pip install aurelio_sdk --no-deps
|
||||
pip uninstall posthog -y
|
||||
- setup_litellm_enterprise_pip
|
||||
- save_cache:
|
||||
paths:
|
||||
- ./venv
|
||||
key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
|
||||
- run:
|
||||
name: Run prisma ./docker/entrypoint.sh
|
||||
command: |
|
||||
set +e
|
||||
chmod +x docker/entrypoint.sh
|
||||
./docker/entrypoint.sh
|
||||
set -e
|
||||
- run:
|
||||
name: Black Formatting
|
||||
command: |
|
||||
cd litellm
|
||||
python -m pip install black
|
||||
python -m black .
|
||||
cd ..
|
||||
|
||||
# Run pytest and generate JUnit XML report
|
||||
- run:
|
||||
name: Run tests (Part 2 - N-Z)
|
||||
command: |
|
||||
mkdir test-results
|
||||
|
||||
# Discover test files (N-Z)
|
||||
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py")
|
||||
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--split-by=timings \
|
||||
--verbose \
|
||||
--command="xargs python -m pytest \
|
||||
-vv \
|
||||
--cov=litellm \
|
||||
--cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=20 \
|
||||
-k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
|
||||
-n 4 \
|
||||
--timeout=300 \
|
||||
--timeout_method=thread"
|
||||
no_output_timeout: 120m
|
||||
- run:
|
||||
name: Rename the coverage files
|
||||
command: |
|
||||
mv coverage.xml local_testing_part2_coverage.xml
|
||||
mv .coverage local_testing_part2_coverage
|
||||
|
||||
# Store test results
|
||||
- store_test_results:
|
||||
path: test-results
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- local_testing_part2_coverage.xml
|
||||
- local_testing_part2_coverage
|
||||
langfuse_logging_unit_tests:
|
||||
docker:
|
||||
- image: cimg/python:3.11
|
||||
|
|
@ -3300,7 +3428,7 @@ jobs:
|
|||
python -m venv venv
|
||||
. venv/bin/activate
|
||||
pip install coverage
|
||||
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
|
||||
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
|
||||
coverage xml
|
||||
- codecov/upload:
|
||||
file: ./coverage.xml
|
||||
|
|
@ -3755,7 +3883,13 @@ workflows:
|
|||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- local_testing:
|
||||
- local_testing_part1:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- local_testing_part2:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
|
|
@ -4060,7 +4194,8 @@ workflows:
|
|||
- litellm_proxy_unit_testing_part2
|
||||
- litellm_security_tests
|
||||
- langfuse_logging_unit_tests
|
||||
- local_testing
|
||||
- local_testing_part1
|
||||
- local_testing_part2
|
||||
- litellm_assistants_api_testing
|
||||
- auth_ui_unit_tests
|
||||
- db_migration_disable_update_check:
|
||||
|
|
@ -4103,7 +4238,8 @@ workflows:
|
|||
- publish_to_pypi:
|
||||
requires:
|
||||
- mypy_linting
|
||||
- local_testing
|
||||
- local_testing_part1
|
||||
- local_testing_part2
|
||||
- build_and_test
|
||||
- e2e_openai_endpoints
|
||||
- test_bad_database_url
|
||||
|
|
|
|||
|
|
@ -9788,6 +9788,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"deepinfra/google/gemini-2.0-flash-001": {
|
||||
"deprecation_date": "2026-03-31",
|
||||
"max_tokens": 1000000,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 1000000,
|
||||
|
|
@ -12105,6 +12106,7 @@
|
|||
},
|
||||
"gemini-2.0-flash": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_audio_token": 7e-07,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -12144,7 +12146,7 @@
|
|||
},
|
||||
"gemini-2.0-flash-001": {
|
||||
"cache_read_input_token_cost": 3.75e-08,
|
||||
"deprecation_date": "2026-02-05",
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -12230,6 +12232,7 @@
|
|||
},
|
||||
"gemini-2.0-flash-lite": {
|
||||
"cache_read_input_token_cost": 1.875e-08,
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_audio_token": 7.5e-08,
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -12265,7 +12268,7 @@
|
|||
},
|
||||
"gemini-2.0-flash-lite-001": {
|
||||
"cache_read_input_token_cost": 1.875e-08,
|
||||
"deprecation_date": "2026-02-25",
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_audio_token": 7.5e-08,
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -13930,6 +13933,7 @@
|
|||
},
|
||||
"gemini/gemini-2.0-flash": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_audio_token": 7e-07,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "gemini",
|
||||
|
|
@ -13970,6 +13974,7 @@
|
|||
},
|
||||
"gemini/gemini-2.0-flash-001": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_audio_token": 7e-07,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "gemini",
|
||||
|
|
@ -14057,6 +14062,7 @@
|
|||
},
|
||||
"gemini/gemini-2.0-flash-lite": {
|
||||
"cache_read_input_token_cost": 1.875e-08,
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_audio_token": 7.5e-08,
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "gemini",
|
||||
|
|
@ -23158,6 +23164,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/google/gemini-2.0-flash-001": {
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_audio_token": 7e-07,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
|
|
@ -27695,6 +27702,7 @@
|
|||
"output_cost_per_token": 9e-07
|
||||
},
|
||||
"vercel_ai_gateway/google/gemini-2.0-flash": {
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "vercel_ai_gateway",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -27704,6 +27712,7 @@
|
|||
"output_cost_per_token": 6e-07
|
||||
},
|
||||
"vercel_ai_gateway/google/gemini-2.0-flash-lite": {
|
||||
"deprecation_date": "2026-03-31",
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "vercel_ai_gateway",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue