diff --git a/.circleci/config.yml b/.circleci/config.yml
index 8672561f654..307247651e7 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -21,43 +21,35 @@ commands:
- run:
name: "Install local version of litellm-enterprise"
command: |
- cd enterprise
- python -m pip install -e .
- cd ..
+ pip install --force-reinstall --no-deps -e enterprise/
setup_litellm_test_deps:
steps:
- checkout
- setup_google_dns
- restore_cache:
keys:
- - v2-litellm-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }}
- - v2-litellm-deps-
+ - v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }}
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
- pip install "pytest-mock==3.12.0"
- pip install "pytest==7.3.1"
- pip install "pytest-retry==1.6.3"
- pip install "pytest-cov==5.0.0"
- pip install "pytest-asyncio==0.21.1"
- pip install "respx==0.22.0"
- pip install "hypercorn==0.17.3"
- pip install "pydantic==2.11.0"
- pip install "mcp==1.25.0"
- pip install "requests-mock>=1.12.1"
- pip install "responses==0.25.7"
- pip install "pytest-xdist==3.6.1"
- pip install "pytest-timeout==2.2.0"
- pip install "semantic_router==0.1.10"
- pip install "fastapi-offline==1.7.3"
- pip install "a2a"
+ python -m pip install --upgrade pip uv
+ # Use uv for the heavy requirements.txt (10-100x faster than pip)
+ uv pip install --system -r requirements.txt
+ # Use pip for test deps (small set, avoids uv strict-resolution
+ # conflicts with transitive dep pins like openai<2 and pydantic>=2.11.5)
+ pip install "pytest-mock==3.12.0" "pytest==7.3.1" "pytest-retry==1.6.3" \
+ "pytest-asyncio==0.21.1" "respx==0.22.0" "hypercorn==0.17.3" \
+ "pydantic==2.12.5" "mcp==1.26.0" "requests-mock>=1.12.1" \
+ "responses==0.25.7" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" \
+ "pytest-cov==5.0.0" "semantic_router==0.1.10" "fastapi-offline==1.7.3" \
+ "a2a" "parameterized>=0.9.0"
- setup_litellm_enterprise_pip
- save_cache:
paths:
- - ~/.cache/pip
- key: v2-litellm-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }}
+ - ~/.local/lib
+ - ~/.local/bin
+ - ~/.cache/uv
+ key: v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }}
jobs:
# Add Windows testing job
@@ -71,9 +63,11 @@ jobs:
- run:
name: Install Python
command: |
- choco install python --version=3.11.0 -y
+ choco install python --version=3.11.0 -y --no-progress --force
refreshenv
python --version
+ environment:
+ CHOCOLATEY_CONFIRM_ALL: "true"
- run:
name: Install Dependencies
command: |
@@ -100,8 +94,8 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
pip uninstall fastuuid -y
pip install "mypy==1.18.2"
- run:
@@ -112,6 +106,25 @@ jobs:
python -m mypy .
cd ..
no_output_timeout: 10m
+
+ semgrep:
+ docker:
+ - image: cimg/python:3.12
+ auth:
+ username: ${DOCKERHUB_USERNAME}
+ password: ${DOCKERHUB_PASSWORD}
+ working_directory: ~/project
+ resource_class: medium
+ steps:
+ - checkout
+ - setup_google_dns
+ - run:
+ name: Install Semgrep
+ command: pip install semgrep
+ - run:
+ name: Run Semgrep (custom rules only)
+ command: semgrep scan --config .semgrep/rules . --error
+
local_testing_part1:
docker:
- image: cimg/python:3.12
@@ -136,50 +149,17 @@ jobs:
command: |
python -m pip install --upgrade pip
python -m pip install -r .circleci/requirements.txt
- pip install "pytest==7.3.1"
- pip install "pytest-retry==1.6.3"
- pip install "pytest-asyncio==0.21.1"
- pip install "pytest-cov==5.0.0"
- pip install "mypy==1.18.2"
- pip install "google-generativeai==0.3.2"
- pip install "google-cloud-aiplatform==1.43.0"
- pip install pyarrow
- pip install "boto3==1.36.0"
- pip install "aioboto3==13.4.0"
- pip install langchain
- pip install lunary==0.2.5
- pip install "azure-identity==1.16.1"
- pip install "langfuse==2.59.7"
- pip install "logfire==0.29.0"
- pip install numpydoc
- pip install traceloop-sdk==0.21.1
- pip install opentelemetry-api==1.25.0
- pip install opentelemetry-sdk==1.25.0
- pip install opentelemetry-exporter-otlp==1.25.0
- pip install openai==1.100.1
- pip install prisma==0.11.0
- pip install "detect_secrets==1.5.0"
- pip install "httpx==0.24.1"
- pip install "respx==0.22.0"
- pip install fastapi
- pip install "gunicorn==21.2.0"
- pip install "anyio==4.2.0"
- pip install "aiodynamo==23.10.1"
- pip install "asyncio==3.4.3"
- pip install "apscheduler==3.10.4"
- pip install "PyGithub==1.59.1"
- pip install argon2-cffi
- pip install "pytest-mock==3.12.0"
- pip install python-multipart
- pip install google-cloud-aiplatform
- pip install prometheus-client==0.20.0
- pip install "pydantic==2.10.2"
- pip install "diskcache==5.6.1"
- pip install "Pillow==10.3.0"
- pip install "jsonschema==4.22.0"
- pip install "pytest-xdist==3.6.1"
- pip install "pytest-timeout==2.2.0"
- pip install "websockets==13.1.0"
+ pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \
+ "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \
+ "boto3==1.42.80" langchain lunary==0.2.5 \
+ "azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \
+ traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \
+ "detect_secrets==1.5.0" "respx==0.22.0" fastapi \
+ "gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \
+ "apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \
+ python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \
+ "diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \
+ "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1"
pip install semantic_router --no-deps
pip install aurelio_sdk --no-deps
pip uninstall posthog -y
@@ -225,7 +205,7 @@ jobs:
-n 4 \
--timeout=300 \
--timeout_method=thread"
- no_output_timeout: 120m
+ no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
@@ -264,50 +244,17 @@ jobs:
command: |
python -m pip install --upgrade pip
python -m pip install -r .circleci/requirements.txt
- pip install "pytest==7.3.1"
- pip install "pytest-retry==1.6.3"
- pip install "pytest-asyncio==0.21.1"
- pip install "pytest-cov==5.0.0"
- pip install "mypy==1.18.2"
- pip install "google-generativeai==0.3.2"
- pip install "google-cloud-aiplatform==1.43.0"
- pip install pyarrow
- pip install "boto3==1.36.0"
- pip install "aioboto3==13.4.0"
- pip install langchain
- pip install lunary==0.2.5
- pip install "azure-identity==1.16.1"
- pip install "langfuse==2.59.7"
- pip install "logfire==0.29.0"
- pip install numpydoc
- pip install traceloop-sdk==0.21.1
- pip install opentelemetry-api==1.25.0
- pip install opentelemetry-sdk==1.25.0
- pip install opentelemetry-exporter-otlp==1.25.0
- pip install openai==1.100.1
- pip install prisma==0.11.0
- pip install "detect_secrets==1.5.0"
- pip install "httpx==0.24.1"
- pip install "respx==0.22.0"
- pip install fastapi
- pip install "gunicorn==21.2.0"
- pip install "anyio==4.2.0"
- pip install "aiodynamo==23.10.1"
- pip install "asyncio==3.4.3"
- pip install "apscheduler==3.10.4"
- pip install "PyGithub==1.59.1"
- pip install argon2-cffi
- pip install "pytest-mock==3.12.0"
- pip install python-multipart
- pip install google-cloud-aiplatform
- pip install prometheus-client==0.20.0
- pip install "pydantic==2.10.2"
- pip install "diskcache==5.6.1"
- pip install "Pillow==10.3.0"
- pip install "jsonschema==4.22.0"
- pip install "pytest-xdist==3.6.1"
- pip install "pytest-timeout==2.2.0"
- pip install "websockets==13.1.0"
+ pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \
+ "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \
+ "boto3==1.42.80" langchain lunary==0.2.5 \
+ "azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \
+ traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \
+ "detect_secrets==1.5.0" "respx==0.22.0" fastapi \
+ "gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \
+ "apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \
+ python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \
+ "diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \
+ "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1"
pip install semantic_router --no-deps
pip install aurelio_sdk --no-deps
pip uninstall posthog -y
@@ -353,7 +300,7 @@ jobs:
-n 4 \
--timeout=300 \
--timeout_method=thread"
- no_output_timeout: 120m
+ no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
@@ -375,6 +322,7 @@ jobs:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
+ resource_class: medium
steps:
- checkout
@@ -398,42 +346,41 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
- pip install "google-cloud-aiplatform==1.43.0"
+ pip install "google-cloud-aiplatform==1.133.0"
pip install pyarrow
- pip install "boto3==1.36.0"
- pip install "aioboto3==13.4.0"
+ pip install "boto3==1.42.80"
pip install langchain
pip install lunary==0.2.5
- pip install "azure-identity==1.16.1"
+ pip install "azure-identity==1.25.3"
pip install "langfuse==2.59.7"
pip install "logfire==0.29.0"
pip install numpydoc
pip install traceloop-sdk==0.21.1
- pip install opentelemetry-api==1.25.0
- pip install opentelemetry-sdk==1.25.0
- pip install opentelemetry-exporter-otlp==1.25.0
+ pip install opentelemetry-api==1.28.0
+ pip install opentelemetry-sdk==1.28.0
+ pip install opentelemetry-exporter-otlp==1.28.0
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
- pip install "httpx==0.24.1"
+ pip install "httpx==0.28.1"
pip install "respx==0.22.0"
pip install fastapi
- pip install "gunicorn==21.2.0"
- pip install "anyio==4.2.0"
+ pip install "gunicorn==23.0.0"
+ pip install "anyio==4.8.0"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
- pip install "apscheduler==3.10.4"
+ pip install "apscheduler==3.11.2"
pip install "PyGithub==1.59.1"
pip install argon2-cffi
pip install "pytest-mock==3.12.0"
pip install python-multipart
pip install google-cloud-aiplatform
pip install prometheus-client==0.20.0
- pip install "pydantic==2.10.2"
+ pip install "pydantic==2.12.5"
pip install "diskcache==5.6.1"
- pip install "Pillow==10.3.0"
- pip install "jsonschema==4.22.0"
- pip install "websockets==13.1.0"
+ pip install "Pillow==12.1.1"
+ pip install "jsonschema==4.23.0"
+ pip install "websockets==15.0.1"
- setup_litellm_enterprise_pip
- save_cache:
paths:
@@ -453,129 +400,11 @@ jobs:
command: |
pwd
ls
- python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse"
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml langfuse_coverage.xml
- mv .coverage langfuse_coverage
-
+ python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse"
+ no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - langfuse_coverage.xml
- - langfuse_coverage
- caching_unit_tests:
- docker:
- - image: cimg/python:3.11
- auth:
- username: ${DOCKERHUB_USERNAME}
- password: ${DOCKERHUB_PASSWORD}
- working_directory: ~/project
-
- steps:
- - checkout
- - setup_google_dns
- - run:
- name: DNS lookup for Redis host
- command: |
- sudo apt-get update
- sudo apt-get install -y dnsutils
- dig redis-19899.c239.us-east-1-2.ec2.redns.redis-cloud.com +short
- - run:
- name: Show git commit hash
- command: |
- echo "Git commit hash: $CIRCLE_SHA1"
-
- - restore_cache:
- keys:
- - v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
- - run:
- name: Install Dependencies
- command: |
- python -m pip install --upgrade pip
- python -m pip install -r .circleci/requirements.txt
- pip install "pytest==7.3.1"
- pip install "pytest-retry==1.6.3"
- pip install "pytest-asyncio==0.21.1"
- pip install "pytest-cov==5.0.0"
- pip install "mypy==1.18.2"
- pip install "google-generativeai==0.3.2"
- pip install "google-cloud-aiplatform==1.43.0"
- pip install pyarrow
- pip install "boto3==1.36.0"
- pip install "aioboto3==13.4.0"
- pip install langchain
- pip install lunary==0.2.5
- pip install "azure-identity==1.16.1"
- pip install "langfuse==2.59.7"
- pip install "logfire==0.29.0"
- pip install numpydoc
- pip install traceloop-sdk==0.21.1
- pip install opentelemetry-api==1.25.0
- pip install opentelemetry-sdk==1.25.0
- pip install opentelemetry-exporter-otlp==1.25.0
- pip install openai==1.100.1
- pip install prisma==0.11.0
- pip install "detect_secrets==1.5.0"
- pip install "httpx==0.24.1"
- pip install "respx==0.22.0"
- pip install fastapi
- pip install "gunicorn==21.2.0"
- pip install "anyio==4.2.0"
- pip install "aiodynamo==23.10.1"
- pip install "asyncio==3.4.3"
- pip install "apscheduler==3.10.4"
- pip install "PyGithub==1.59.1"
- pip install argon2-cffi
- pip install "pytest-mock==3.12.0"
- pip install python-multipart
- pip install google-cloud-aiplatform
- pip install prometheus-client==0.20.0
- pip install "pydantic==2.10.2"
- pip install "diskcache==5.6.1"
- pip install "Pillow==10.3.0"
- pip install "jsonschema==4.22.0"
- pip install "websockets==13.1.0"
- - setup_litellm_enterprise_pip
- - save_cache:
- paths:
- - ./venv
- key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
- - run:
- name: Run prisma ./docker/entrypoint.sh
- command: |
- set +e
- chmod +x docker/entrypoint.sh
- ./docker/entrypoint.sh
- set -e
-
- # Run pytest and generate JUnit XML report
- - run:
- name: Run tests
- command: |
- pwd
- ls
- python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "caching or cache"
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml caching_coverage.xml
- mv .coverage caching_coverage
-
- # Store test results
- - store_test_results:
- path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - caching_coverage.xml
- - caching_coverage
auth_ui_unit_tests:
docker:
- image: cimg/python:3.11
@@ -590,12 +419,12 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
- pip install "pytest-cov==5.0.0"
+ pip install "pytest-xdist==3.6.1"
- save_cache:
paths:
- ./venv
@@ -613,25 +442,13 @@ jobs:
command: |
pwd
ls
- python -m pytest -vv tests/proxy_admin_ui_tests -x --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
-
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml auth_ui_unit_tests_coverage.xml
- mv .coverage auth_ui_unit_tests_coverage
+ python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2
+ no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - auth_ui_unit_tests_coverage.xml
- - auth_ui_unit_tests_coverage
-
litellm_router_testing: # Runs all tests with the "router" keyword
docker:
- image: cimg/python:3.11
@@ -639,22 +456,32 @@ jobs:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
+ resource_class: large
+ parallelism: 4
steps:
- checkout
- setup_google_dns
+ - restore_cache:
+ keys:
+ - v1-router-testing-deps-{{ checksum "requirements.txt" }}
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
pip install "pytest==7.3.1"
pip install "respx==0.22.0"
- pip install "pytest-cov==5.0.0"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-xdist==3.6.1"
+ pip install "pytest-timeout==2.2.0"
pip install semantic_router --no-deps
pip install aurelio_sdk --no-deps
+ - save_cache:
+ paths:
+ - /home/circleci/.pyenv
+ - /home/circleci/.local
+ key: v1-router-testing-deps-{{ checksum "requirements.txt" }}
# Run pytest and generate JUnit XML report
- setup_litellm_enterprise_pip
- run:
@@ -662,23 +489,23 @@ jobs:
command: |
pwd
ls
- python -m pytest tests/local_testing --cov=litellm --cov-report=xml -vv -k "router" -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml litellm_router_coverage.xml
- mv .coverage litellm_router_coverage
+ TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
+
+ echo "$TEST_FILES" | circleci tests run \
+ --split-by=timings \
+ --verbose \
+ --command="xargs python -m pytest \
+ -v \
+ -k 'router' \
+ -n 4 \
+ --junitxml=test-results/junit.xml \
+ --durations=5 \
+ --timeout=300 --timeout_method=thread"
+ no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - litellm_router_coverage.xml
- - litellm_router_coverage
-
litellm_router_unit_testing: # Runs all tests with the "router" keyword
docker:
- image: cimg/python:3.11
@@ -686,23 +513,31 @@ jobs:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
+ resource_class: large
steps:
- checkout
- setup_google_dns
+ - restore_cache:
+ keys:
+ - v1-router-unit-deps-{{ checksum "requirements.txt" }}
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
pip install "pytest==7.3.1"
pip install "respx==0.22.0"
- pip install "pytest-cov==5.0.0"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install semantic_router --no-deps
pip install aurelio_sdk --no-deps
pip install "pytest-xdist==3.6.1"
+ - save_cache:
+ paths:
+ - /home/circleci/.pyenv
+ - /home/circleci/.local
+ key: v1-router-unit-deps-{{ checksum "requirements.txt" }}
# Run pytest and generate JUnit XML report
- setup_litellm_enterprise_pip
- run:
@@ -710,475 +545,11 @@ jobs:
command: |
pwd
ls
- python -m pytest -vv tests/router_unit_tests --cov=litellm --cov-report=xml -x -s --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml litellm_router_unit_coverage.xml
- mv .coverage litellm_router_unit_coverage
+ python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4
+ no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
-
- - persist_to_workspace:
- root: .
- paths:
- - litellm_router_unit_coverage.xml
- - litellm_router_unit_coverage
- litellm_security_tests:
- machine:
- image: ubuntu-2204:2023.10.1
- resource_class: xlarge
- working_directory: ~/project
- steps:
- - checkout
- - setup_google_dns
- - run:
- name: Show git commit hash
- command: |
- echo "Git commit hash: $CIRCLE_SHA1"
- - run:
- name: Install Docker CLI (In case it's not already installed)
- command: |
- curl -fsSL https://get.docker.com | sh
- sudo usermod -aG docker $USER
- docker version
- - run:
- name: Install Python 3.13
- command: |
- curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
- bash miniconda.sh -b -p $HOME/miniconda
- export PATH="$HOME/miniconda/bin:$PATH"
- conda init bash
- source ~/.bashrc
- conda create -n myenv python=3.13 -y
- conda activate myenv
- python --version
- - run:
- name: Install Dependencies
- command: |
- export PATH="$HOME/miniconda/bin:$PATH"
- source $HOME/miniconda/etc/profile.d/conda.sh
- conda activate myenv
- python --version
- which python
- pip install --upgrade typing-extensions>=4.12.0
- pip install "pytest==7.3.1"
- pip install "pytest-asyncio==0.21.1"
- pip install aiohttp
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
- pip install "pytest==7.3.1"
- pip install "pytest-retry==1.6.3"
- pip install "pytest-mock==3.12.0"
- pip install "pytest-asyncio==0.21.1"
- pip install "mypy==1.18.2"
- pip install "google-generativeai==0.3.2"
- pip install "google-cloud-aiplatform==1.43.0"
- pip install pyarrow
- pip install "boto3==1.36.0"
- pip install "aioboto3==13.4.0"
- pip install langchain
- pip install "langfuse>=2.0.0"
- pip install "logfire==0.29.0"
- pip install numpydoc
- pip install prisma
- pip install fastapi
- pip install jsonschema
- pip install "httpx==0.24.1"
- pip install "gunicorn==21.2.0"
- pip install "anyio==3.7.1"
- pip install "aiodynamo==23.10.1"
- pip install "asyncio==3.4.3"
- pip install "PyGithub==1.59.1"
- pip install "openai==1.100.1"
- pip install "pytest-cov==5.0.0"
- pip install "apscheduler"
- - run:
- name: Install dockerize
- command: |
- wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
- sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
- rm dockerize-linux-amd64-v0.6.1.tar.gz
- - run:
- name: Start PostgreSQL Database
- command: |
- docker run -d \
- --name postgres-db \
- -e POSTGRES_USER=postgres \
- -e POSTGRES_PASSWORD=postgres \
- -e POSTGRES_DB=circle_test \
- -p 5432:5432 \
- postgres:14
- - run:
- name: Wait for PostgreSQL to be ready
- command: dockerize -wait tcp://localhost:5432 -timeout 1m
- - run:
- name: Set DATABASE_URL environment variable
- command: |
- echo 'export DATABASE_URL="postgresql://postgres:postgres@localhost:5432/circle_test"' >> $BASH_ENV
- source $BASH_ENV
- - run:
- name: Run Security Scans
- command: |
- chmod +x ci_cd/security_scans.sh
- ./ci_cd/security_scans.sh
- - run:
- name: Run prisma ./docker/entrypoint.sh
- command: |
- export PATH="$HOME/miniconda/bin:$PATH"
- source $HOME/miniconda/etc/profile.d/conda.sh
- conda activate myenv
- set +e
- chmod +x docker/entrypoint.sh
- ./docker/entrypoint.sh
- set -e
- # Run pytest and generate JUnit XML report
- - run:
- name: Run tests
- command: |
- export PATH="$HOME/miniconda/bin:$PATH"
- source $HOME/miniconda/etc/profile.d/conda.sh
- conda activate myenv
- pwd
- ls
- python -m pytest tests/proxy_security_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml litellm_security_tests_coverage.xml
- mv .coverage litellm_security_tests_coverage
- # Store test results
- - store_test_results:
- path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - litellm_security_tests_coverage.xml
- - litellm_security_tests_coverage
- # Split proxy unit tests into 3 jobs for faster execution and better debugging
- # test_key_generate_prisma runs separately without parallel execution to avoid event loop issues with logging worker
- litellm_proxy_unit_testing_key_generation:
- docker:
- - image: cimg/python:3.11
- auth:
- username: ${DOCKERHUB_USERNAME}
- password: ${DOCKERHUB_PASSWORD}
- working_directory: ~/project
- resource_class: large
- steps:
- - checkout
- - setup_google_dns
- - run:
- name: Show git commit hash
- command: |
- echo "Git commit hash: $CIRCLE_SHA1"
- - run:
- name: Install PostgreSQL
- command: |
- sudo apt-get update
- sudo apt-get install -y postgresql-14 postgresql-contrib-14
- - restore_cache:
- keys:
- - v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
- - run:
- name: Install Dependencies
- command: |
- python -m pip install --upgrade pip
- python -m pip install -r .circleci/requirements.txt
- pip install "pytest==7.3.1"
- pip install "pytest-retry==1.6.3"
- pip install "pytest-asyncio==0.21.1"
- pip install "pytest-cov==5.0.0"
- pip install "pytest-timeout==2.2.0"
- pip install "pytest-forked==1.6.0"
- pip install "mypy==1.18.2"
- pip install "google-generativeai==0.3.2"
- pip install "google-cloud-aiplatform==1.43.0"
- pip install "google-genai==1.22.0"
- pip install pyarrow
- pip install "boto3==1.36.0"
- pip install "aioboto3==13.4.0"
- pip install langchain
- pip install lunary==0.2.5
- pip install "azure-identity==1.16.1"
- pip install "langfuse==2.59.7"
- pip install "logfire==0.29.0"
- pip install numpydoc
- pip install traceloop-sdk==0.21.1
- pip install opentelemetry-api==1.25.0
- pip install opentelemetry-sdk==1.25.0
- pip install opentelemetry-exporter-otlp==1.25.0
- pip install openai==1.100.1
- pip install prisma==0.11.0
- pip install "detect_secrets==1.5.0"
- pip install "httpx==0.24.1"
- pip install "respx==0.22.0"
- pip install fastapi
- pip install "gunicorn==21.2.0"
- pip install "anyio==4.2.0"
- pip install "aiodynamo==23.10.1"
- pip install "asyncio==3.4.3"
- pip install "apscheduler==3.10.4"
- pip install "PyGithub==1.59.1"
- pip install argon2-cffi
- pip install "pytest-mock==3.12.0"
- pip install python-multipart
- pip install google-cloud-aiplatform
- pip install prometheus-client==0.20.0
- pip install "pydantic==2.10.2"
- pip install "diskcache==5.6.1"
- pip install "Pillow==10.3.0"
- pip install "jsonschema==4.22.0"
- pip install "pytest-postgresql==7.0.1"
- pip install "fakeredis==2.28.1"
- - setup_litellm_enterprise_pip
- - save_cache:
- paths:
- - ./venv
- key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
- - run:
- name: Run prisma ./docker/entrypoint.sh
- command: |
- set +e
- chmod +x docker/entrypoint.sh
- ./docker/entrypoint.sh
- set -e
- - run:
- name: Run key generation tests (no parallel execution to avoid event loop issues)
- command: |
- pwd
- ls
- # Run without -n flag to avoid pytest-xdist event loop conflicts with logging worker
- python -m pytest tests/proxy_unit_tests/test_key_generate_prisma.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-key-generation.xml --durations=10 --timeout=300 -vv --log-cli-level=INFO
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml litellm_proxy_unit_tests_key_generation_coverage.xml
- mv .coverage litellm_proxy_unit_tests_key_generation_coverage
- - store_test_results:
- path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - litellm_proxy_unit_tests_key_generation_coverage.xml
- - litellm_proxy_unit_tests_key_generation_coverage
- litellm_proxy_unit_testing_part1:
- docker:
- - image: cimg/python:3.11
- auth:
- username: ${DOCKERHUB_USERNAME}
- password: ${DOCKERHUB_PASSWORD}
- working_directory: ~/project
- resource_class: large
- steps:
- - checkout
- - setup_google_dns
- - run:
- name: Show git commit hash
- command: |
- echo "Git commit hash: $CIRCLE_SHA1"
- - run:
- name: Install PostgreSQL
- command: |
- sudo apt-get update
- sudo apt-get install -y postgresql-14 postgresql-contrib-14
- - restore_cache:
- keys:
- - v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
- - run:
- name: Install Dependencies
- command: |
- python -m pip install --upgrade pip
- python -m pip install -r .circleci/requirements.txt
- pip install "pytest==7.3.1"
- pip install "pytest-retry==1.6.3"
- pip install "pytest-asyncio==0.21.1"
- pip install "pytest-cov==5.0.0"
- pip install "pytest-timeout==2.2.0"
- pip install "pytest-forked==1.6.0"
- pip install "mypy==1.18.2"
- pip install "google-generativeai==0.3.2"
- pip install "google-cloud-aiplatform==1.43.0"
- pip install "google-genai==1.22.0"
- pip install pyarrow
- pip install "boto3==1.36.0"
- pip install "aioboto3==13.4.0"
- pip install langchain
- pip install lunary==0.2.5
- pip install "azure-identity==1.16.1"
- pip install "langfuse==2.59.7"
- pip install "logfire==0.29.0"
- pip install numpydoc
- pip install traceloop-sdk==0.21.1
- pip install opentelemetry-api==1.25.0
- pip install opentelemetry-sdk==1.25.0
- pip install opentelemetry-exporter-otlp==1.25.0
- pip install openai==1.100.1
- pip install prisma==0.11.0
- pip install "detect_secrets==1.5.0"
- pip install "httpx==0.24.1"
- pip install "respx==0.22.0"
- pip install fastapi
- pip install "gunicorn==21.2.0"
- pip install "anyio==4.2.0"
- pip install "aiodynamo==23.10.1"
- pip install "asyncio==3.4.3"
- pip install "apscheduler==3.10.4"
- pip install "PyGithub==1.59.1"
- pip install argon2-cffi
- pip install "pytest-mock==3.12.0"
- pip install python-multipart
- pip install google-cloud-aiplatform
- pip install prometheus-client==0.20.0
- pip install "pydantic==2.10.2"
- pip install "diskcache==5.6.1"
- pip install "Pillow==10.3.0"
- pip install "jsonschema==4.22.0"
- pip install "pytest-postgresql==7.0.1"
- pip install "fakeredis==2.28.1"
- pip install "pytest-xdist==3.6.1"
- - setup_litellm_enterprise_pip
- - save_cache:
- paths:
- - ./venv
- key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
- - run:
- name: Run prisma ./docker/entrypoint.sh
- command: |
- set +e
- chmod +x docker/entrypoint.sh
- ./docker/entrypoint.sh
- set -e
- - run:
- name: Run proxy unit tests (part 1 - auth checks only, key generation in separate job)
- command: |
- pwd
- ls
- # Run auth tests with parallel execution (test_key_generate_prisma moved to separate job to avoid event loop issues)
- python -m pytest tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part1.xml --durations=10 -n 8 --timeout=300 -vv --log-cli-level=INFO
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml litellm_proxy_unit_tests_part1_coverage.xml
- mv .coverage litellm_proxy_unit_tests_part1_coverage
- - store_test_results:
- path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - litellm_proxy_unit_tests_part1_coverage.xml
- - litellm_proxy_unit_tests_part1_coverage
- litellm_proxy_unit_testing_part2:
- docker:
- - image: cimg/python:3.11
- auth:
- username: ${DOCKERHUB_USERNAME}
- password: ${DOCKERHUB_PASSWORD}
- working_directory: ~/project
- resource_class: large
- steps:
- - checkout
- - setup_google_dns
- - run:
- name: Show git commit hash
- command: |
- echo "Git commit hash: $CIRCLE_SHA1"
- - run:
- name: Install PostgreSQL
- command: |
- sudo apt-get update
- sudo apt-get install -y postgresql-14 postgresql-contrib-14
- - restore_cache:
- keys:
- - v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
- - run:
- name: Install Dependencies
- command: |
- python -m pip install --upgrade pip
- python -m pip install -r .circleci/requirements.txt
- pip install "pytest==7.3.1"
- pip install "pytest-retry==1.6.3"
- pip install "pytest-asyncio==0.21.1"
- pip install "pytest-cov==5.0.0"
- pip install "pytest-timeout==2.2.0"
- pip install "pytest-forked==1.6.0"
- pip install "mypy==1.18.2"
- pip install "google-generativeai==0.3.2"
- pip install "google-cloud-aiplatform==1.43.0"
- pip install "google-genai==1.22.0"
- pip install pyarrow
- pip install "boto3==1.36.0"
- pip install "aioboto3==13.4.0"
- pip install langchain
- pip install lunary==0.2.5
- pip install "azure-identity==1.16.1"
- pip install "langfuse==2.59.7"
- pip install "logfire==0.29.0"
- pip install numpydoc
- pip install traceloop-sdk==0.21.1
- pip install opentelemetry-api==1.25.0
- pip install opentelemetry-sdk==1.25.0
- pip install opentelemetry-exporter-otlp==1.25.0
- pip install openai==1.100.1
- pip install prisma==0.11.0
- pip install "detect_secrets==1.5.0"
- pip install "httpx==0.24.1"
- pip install "respx==0.22.0"
- pip install fastapi
- pip install "gunicorn==21.2.0"
- pip install "anyio==4.2.0"
- pip install "aiodynamo==23.10.1"
- pip install "asyncio==3.4.3"
- pip install "apscheduler==3.10.4"
- pip install "PyGithub==1.59.1"
- pip install argon2-cffi
- pip install "pytest-mock==3.12.0"
- pip install python-multipart
- pip install google-cloud-aiplatform
- pip install prometheus-client==0.20.0
- pip install "pydantic==2.10.2"
- pip install "diskcache==5.6.1"
- pip install "Pillow==10.3.0"
- pip install "jsonschema==4.22.0"
- pip install "pytest-postgresql==7.0.1"
- pip install "fakeredis==2.28.1"
- pip install "pytest-xdist==3.6.1"
- - setup_litellm_enterprise_pip
- - save_cache:
- paths:
- - ./venv
- key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
- - run:
- name: Run prisma ./docker/entrypoint.sh
- command: |
- set +e
- chmod +x docker/entrypoint.sh
- ./docker/entrypoint.sh
- set -e
- - run:
- name: Run proxy unit tests (part 2 - remaining tests)
- command: |
- pwd
- ls
- python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part2.xml --durations=10 -n 8 --timeout=300 -vv --log-cli-level=INFO
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml litellm_proxy_unit_tests_part2_coverage.xml
- mv .coverage litellm_proxy_unit_tests_part2_coverage
- - store_test_results:
- path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - litellm_proxy_unit_tests_part2_coverage.xml
- - litellm_proxy_unit_tests_part2_coverage
litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
docker:
- image: cimg/python:3.13.1
@@ -1186,6 +557,7 @@ jobs:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
+ resource_class: medium
steps:
- checkout
@@ -1193,15 +565,13 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- pip install wheel
- pip install --upgrade pip wheel setuptools
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ pip install wheel setuptools
+ uv pip install --system -r requirements.txt
pip install "pytest==7.3.1"
pip install "respx==0.22.0"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
- pip install "pytest-cov==5.0.0"
# Run pytest and generate JUnit XML report
- setup_litellm_enterprise_pip
- run:
@@ -1209,21 +579,11 @@ jobs:
command: |
pwd
ls
- python -m pytest tests/local_testing/ -vv -k "assistants" --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml litellm_assistants_api_coverage.xml
- mv .coverage litellm_assistants_api_coverage
+ python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5
+ no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - litellm_assistants_api_coverage.xml
- - litellm_assistants_api_coverage
llm_translation_testing:
docker:
- image: cimg/python:3.11
@@ -1231,22 +591,30 @@ jobs:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
+ resource_class: large
steps:
- checkout
- setup_google_dns
+ - restore_cache:
+ keys:
+ - v1-llm-translation-deps-{{ checksum "requirements.txt" }}
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
- pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pytest-xdist==3.6.1"
pip install "pytest-timeout==2.2.0"
+ - save_cache:
+ paths:
+ - /home/circleci/.pyenv
+ - /home/circleci/.local
+ key: v1-llm-translation-deps-{{ checksum "requirements.txt" }}
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@@ -1263,22 +631,12 @@ jobs:
for dir in "${IGNORE_DIRS[@]}"; do
IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir"
done
- python -m pytest -vv tests/llm_translation $IGNORE_ARGS --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml llm_translation_coverage.xml
- mv .coverage llm_translation_coverage
+ python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5
+ no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - llm_translation_coverage.xml
- - llm_translation_coverage
realtime_translation_testing:
docker:
- image: cimg/python:3.11
@@ -1293,16 +651,9 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
- pip install "pytest==7.3.1"
- pip install "pytest-retry==1.6.3"
- pip install "pytest-cov==5.0.0"
- pip install "pytest-asyncio==0.21.1"
- pip install "respx==0.22.0"
- pip install "pytest-xdist==3.6.1"
- pip install "pytest-timeout==2.2.0"
- pip install "websockets"
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
+ pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets"
# Run pytest and generate JUnit XML report
- run:
name: Run realtime tests
@@ -1312,7 +663,7 @@ jobs:
# Add --timeout to kill hanging tests after 120s (2 min)
# Add --durations=20 to show 20 slowest tests for debugging
python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
- no_output_timeout: 120m
+ no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
@@ -1341,23 +692,24 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
- pip install "pydantic==2.11.0"
- pip install "mcp==1.25.0"
+ pip install "pydantic==2.12.5"
+ pip install "mcp==1.26.0"
+ pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
- python -m pytest -vv tests/mcp_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
+ python -m pytest -vv tests/mcp_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
+ no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
@@ -1372,6 +724,51 @@ jobs:
paths:
- mcp_coverage.xml
- mcp_coverage
+ agent_testing:
+ docker:
+ - image: cimg/python:3.11
+ auth:
+ username: ${DOCKERHUB_USERNAME}
+ password: ${DOCKERHUB_PASSWORD}
+ working_directory: ~/project
+
+ steps:
+ - checkout
+ - setup_google_dns
+ - run:
+ name: Install Dependencies
+ command: |
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
+ pip install "pytest==7.3.1"
+ pip install "pytest-retry==1.6.3"
+ pip install "pytest-cov==5.0.0"
+ pip install "pytest-asyncio==0.21.1"
+ pip install "respx==0.22.0"
+ pip install "pydantic==2.12.5"
+ pip install "a2a-sdk"
+ # Run pytest and generate JUnit XML report
+ - run:
+ name: Run tests
+ command: |
+ pwd
+ ls
+ python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+ no_output_timeout: 15m
+ - run:
+ name: Rename the coverage files
+ command: |
+ mv coverage.xml agent_coverage.xml
+ mv .coverage agent_coverage
+
+ # Store test results
+ - store_test_results:
+ path: test-results
+ - persist_to_workspace:
+ root: .
+ paths:
+ - agent_coverage.xml
+ - agent_coverage
guardrails_testing:
docker:
- image: cimg/python:3.11
@@ -1386,23 +783,27 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
- pip install "pydantic==2.10.2"
- pip install "boto3==1.36.0"
+ pip install "pydantic==2.12.5"
+ pip install "boto3==1.42.80"
+ pip install "semantic_router==0.1.10" --no-deps
+ pip install aurelio_sdk
+ pip install "pytest-xdist==3.6.1"
+ pip install "pytest-timeout==2.2.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
- python -m pytest -vv tests/guardrails_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
+ LITELLM_LOG=WARNING python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread
+ no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
@@ -1432,22 +833,22 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
- pip install "pydantic==2.10.2"
+ pip install "pydantic==2.12.5"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
- python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
+ python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5
+ no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
@@ -1470,42 +871,41 @@ jobs:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
+ resource_class: large
steps:
- checkout
- setup_google_dns
+ - restore_cache:
+ keys:
+ - v1-llm-responses-deps-{{ checksum "requirements.txt" }}
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
- pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
+ pip install "pytest-xdist==3.6.1"
+ - save_cache:
+ paths:
+ - /home/circleci/.pyenv
+ - /home/circleci/.local
+ key: v1-llm-responses-deps-{{ checksum "requirements.txt" }}
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
- python -m pytest -vv tests/llm_responses_api_testing --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml llm_responses_api_coverage.xml
- mv .coverage llm_responses_api_coverage
+ python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8
+ no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - llm_responses_api_coverage.xml
- - llm_responses_api_coverage
ocr_testing:
docker:
- image: cimg/python:3.11
@@ -1520,21 +920,17 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
- pip install "pytest==7.3.1"
- pip install "pytest-retry==1.6.3"
- pip install "pytest-cov==5.0.0"
- pip install "pytest-asyncio==0.21.1"
- pip install "respx==0.22.0"
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
+ pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
- python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
+ python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
+ no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
@@ -1563,21 +959,17 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
- pip install "pytest==7.3.1"
- pip install "pytest-retry==1.6.3"
- pip install "pytest-cov==5.0.0"
- pip install "pytest-asyncio==0.21.1"
- pip install "respx==0.22.0"
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
+ pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
- python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
+ python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
+ no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
@@ -1592,143 +984,45 @@ jobs:
paths:
- search_coverage.xml
- search_coverage
- # Split litellm_mapped_tests into 3 parallel jobs for 3x faster execution
- litellm_mapped_tests_proxy:
+ # Split litellm_mapped_tests into parallel jobs
+ litellm_mapped_tests_proxy_part1:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
- resource_class: xlarge
+ resource_class: large
steps:
- setup_litellm_test_deps
- run:
- name: Run proxy tests
+ name: Run proxy tests part 1 (high-volume directories)
command: |
prisma generate
- python -m pytest tests/test_litellm/proxy --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml litellm_proxy_tests_coverage.xml
- mv .coverage litellm_proxy_tests_coverage
+ export PYTHONUNBUFFERED=1
+ python -m pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/client tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part1.xml --durations=10 -n 4 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A
+ no_output_timeout: 15m
- store_test_results:
path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - litellm_proxy_tests_coverage.xml
- - litellm_proxy_tests_coverage
- litellm_mapped_tests_llms:
+ litellm_mapped_tests_proxy_part2:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
- resource_class: xlarge
+ resource_class: large
steps:
- setup_litellm_test_deps
- run:
- name: Run LLM provider tests
+ name: Run proxy tests part 2 (all other tests)
command: |
- python -m pytest tests/test_litellm/llms --cov=litellm --cov-report=xml --junitxml=test-results/junit-llms.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml litellm_llms_tests_coverage.xml
- mv .coverage litellm_llms_tests_coverage
+ prisma generate
+ export PYTHONUNBUFFERED=1
+ python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 4 --maxfail=5 --timeout=120 -vv --log-cli-level=WARNING -r A
+ no_output_timeout: 15m
- store_test_results:
path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - litellm_llms_tests_coverage.xml
- - litellm_llms_tests_coverage
- litellm_mapped_tests_core:
- docker:
- - image: cimg/python:3.11
- auth:
- username: ${DOCKERHUB_USERNAME}
- password: ${DOCKERHUB_PASSWORD}
- working_directory: ~/project
- resource_class: xlarge
- steps:
- - setup_litellm_test_deps
- - run:
- name: Run core tests
- command: |
- python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml litellm_core_tests_coverage.xml
- mv .coverage litellm_core_tests_coverage
- - store_test_results:
- path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - litellm_core_tests_coverage.xml
- - litellm_core_tests_coverage
- litellm_mapped_tests_litellm_core_utils:
- docker:
- - image: cimg/python:3.11
- auth:
- username: ${DOCKERHUB_USERNAME}
- password: ${DOCKERHUB_PASSWORD}
- working_directory: ~/project
- resource_class: xlarge
- steps:
- - setup_litellm_test_deps
- - run:
- name: Run litellm_core_utils tests
- command: |
- python -m pytest tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-litellm-core-utils.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml litellm_core_utils_tests_coverage.xml
- mv .coverage litellm_core_utils_tests_coverage
- - store_test_results:
- path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - litellm_core_utils_tests_coverage.xml
- - litellm_core_utils_tests_coverage
- litellm_mapped_tests_integrations:
- docker:
- - image: cimg/python:3.11
- auth:
- username: ${DOCKERHUB_USERNAME}
- password: ${DOCKERHUB_PASSWORD}
- working_directory: ~/project
- resource_class: xlarge
- steps:
- - setup_litellm_test_deps
- - run:
- name: Run integrations tests
- command: |
- python -m pytest tests/test_litellm/integrations --cov=litellm --cov-report=xml --junitxml=test-results/junit-integrations.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml litellm_integrations_tests_coverage.xml
- mv .coverage litellm_integrations_tests_coverage
- - store_test_results:
- path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - litellm_integrations_tests_coverage.xml
- - litellm_integrations_tests_coverage
litellm_mapped_enterprise_tests:
docker:
- image: cimg/python:3.11
@@ -1736,6 +1030,7 @@ jobs:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
+ resource_class: large
steps:
- checkout
@@ -1743,8 +1038,8 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
pip install "pytest-mock==3.12.0"
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
@@ -1752,12 +1047,13 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "hypercorn==0.17.3"
- pip install "pydantic==2.11.0"
- pip install "mcp==1.25.0"
+ pip install "pydantic==2.12.5"
+ pip install "mcp==1.26.0"
pip install "requests-mock>=1.12.1"
pip install "responses==0.25.7"
pip install "pytest-xdist==3.6.1"
- pip install "semantic_router==0.1.10"
+ pip install "semantic_router==0.1.10" --no-deps
+ pip install aurelio_sdk
pip install "fastapi-offline==1.7.3"
- setup_litellm_enterprise_pip
- run:
@@ -1766,22 +1062,11 @@ jobs:
pwd
ls
prisma generate
- python -m pytest -vv tests/enterprise --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit-enterprise.xml --durations=10 -n 8
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml litellm_mapped_tests_coverage.xml
- mv .coverage litellm_mapped_tests_coverage
-
+ python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4
+ no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - litellm_mapped_tests_coverage.xml
- - litellm_mapped_tests_coverage
batches_testing:
docker:
- image: cimg/python:3.11
@@ -1796,23 +1081,24 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
pip install "respx==0.22.0"
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "google-generativeai==0.3.2"
- pip install "google-cloud-aiplatform==1.43.0"
+ pip install "google-cloud-aiplatform==1.133.0"
+ pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
- python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
+ python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
+ no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
@@ -1841,25 +1127,26 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
pip install numpydoc
- python -m pip install -r requirements.txt
pip install "respx==0.22.0"
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "google-generativeai==0.3.2"
- pip install "google-cloud-aiplatform==1.43.0"
+ pip install "google-cloud-aiplatform==1.133.0"
pip install pytest-mock
+ pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
- python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
+ python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
+ no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
@@ -1889,21 +1176,17 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
- pip install "pytest==7.3.1"
- pip install "pytest-retry==1.6.3"
- pip install "pytest-cov==5.0.0"
- pip install "pytest-asyncio==0.21.1"
- pip install "respx==0.22.0"
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
+ pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
- python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
+ python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
+ no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
@@ -1925,6 +1208,7 @@ jobs:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
+ resource_class: large
steps:
- checkout
@@ -1932,8 +1216,8 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
@@ -1946,22 +1230,11 @@ jobs:
command: |
pwd
ls
- python -m pytest -vv tests/image_gen_tests -n 4 --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
- - run:
- name: Rename the coverage files
- command: |
- mv coverage.xml image_gen_coverage.xml
- mv .coverage image_gen_coverage
-
+ python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5
+ no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
- - persist_to_workspace:
- root: .
- paths:
- - image_gen_coverage.xml
- - image_gen_coverage
logging_testing:
docker:
- image: cimg/python:3.11
@@ -1976,8 +1249,8 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
@@ -1985,11 +1258,12 @@ jobs:
pip install pytest-mock
pip install "respx==0.22.0"
pip install "google-generativeai==0.3.2"
- pip install "google-cloud-aiplatform==1.43.0"
+ pip install "google-cloud-aiplatform==1.133.0"
pip install "mlflow==2.17.2"
- pip install "anthropic==0.52.0"
+ pip install "anthropic==0.54.0"
pip install "blockbuster==1.5.24"
pip install "pytest-xdist==3.6.1"
+ pip install "pytest-timeout==2.2.0"
# Run pytest and generate JUnit XML report
- setup_litellm_enterprise_pip
- run:
@@ -1997,13 +1271,13 @@ jobs:
command: |
pwd
ls
- python -m pytest -vv tests/logging_callback_tests --cov=litellm -n 4 --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
+ LITELLM_LOG=WARNING python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread
+ no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
- mv coverage.xml logging_coverage.xml
- mv .coverage logging_coverage
+ mv coverage.xml logging_coverage.xml || true
+ mv .coverage logging_coverage || true
# Store test results
- store_test_results:
@@ -2027,8 +1301,8 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ uv pip install --system -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
@@ -2041,7 +1315,7 @@ jobs:
pwd
ls
python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
+ no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
@@ -2077,11 +1351,11 @@ jobs:
pip install aiohttp
pip install openai
pip install click
- pip install "boto3==1.36.0"
+ pip install "boto3==1.42.80"
pip install jinja2
- pip install "tokenizers==0.20.0"
+ pip install "tokenizers==0.22.2"
pip install "uvloop==0.21.0"
- pip install "fastuuid==0.12.0"
+ pip install "fastuuid==0.14.0"
pip install jsonschema
- setup_litellm_enterprise_pip
- run:
@@ -2098,6 +1372,7 @@ jobs:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
+ resource_class: medium
steps:
- checkout
@@ -2105,21 +1380,21 @@ jobs:
- run:
name: Install Dependencies
command: |
- python -m pip install --upgrade pip
- python -m pip install wheel setuptools
- python -m pip install -r requirements.txt
+ python -m pip install --upgrade pip uv
+ pip install wheel setuptools
+ uv pip install --system -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "tomli==2.2.1"
- pip install "mcp==1.25.0"
+ pip install "mcp==1.26.0"
- run:
name: Run tests
command: |
pwd
ls
- python -m pytest -vv tests/local_testing/test_basic_python_version.py
+ python -m pytest -v tests/local_testing/test_basic_python_version.py
helm_chart_testing:
machine:
image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker
@@ -2128,6 +1403,8 @@ jobs:
steps:
- checkout
+ - attach_workspace:
+ at: ~/project
- setup_google_dns
# Install Helm
- run:
@@ -2158,10 +1435,11 @@ jobs:
kind create cluster --name litellm-test
- run:
- name: Build Docker image for helm tests
+ name: Load Docker Database Image for helm tests
command: |
+ zstd -d litellm-docker-database.tar.zst --stdout | docker load
IMAGE_TAG=${CIRCLE_SHA1:-ci}
- docker build -t litellm-ci:${IMAGE_TAG} -f docker/Dockerfile.database .
+ docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG}
- run:
name: Load Docker image into Kind
@@ -2232,6 +1510,7 @@ jobs:
- run: python ./tests/code_coverage_tests/router_code_coverage.py
- run: python ./tests/code_coverage_tests/test_chat_completion_imports.py
- run: python ./tests/code_coverage_tests/info_log_check.py
+ - run: python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py
- run: python ./tests/code_coverage_tests/test_ban_set_verbose.py
- run: python ./tests/code_coverage_tests/code_qa_check_tests.py
- run: python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
@@ -2256,7 +1535,7 @@ jobs:
db_migration_disable_update_check:
machine:
image: ubuntu-2204:2023.10.1
- resource_class: xlarge
+ resource_class: medium
working_directory: ~/project
steps:
- checkout
@@ -2279,37 +1558,68 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
pip install apscheduler
+ - run:
+ name: Install dockerize
+ command: |
+ sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+ sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+ sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
+ - run:
+ name: Start PostgreSQL Database
+ command: |
+ docker run -d \
+ --name postgres-db \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_PASSWORD=postgres \
+ -e POSTGRES_DB=litellm_test \
+ -p 5432:5432 \
+ postgres:14
+ - run:
+ name: Wait for PostgreSQL to be ready
+ command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Load Docker Database Image
command: |
- gunzip -c litellm-docker-database.tar.gz | docker load
+ zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
- name: Run Docker container
+ name: Seed database with real schema
+ command: |
+ docker run -d \
+ -p 4001:4000 \
+ -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
+ -e LITELLM_MASTER_KEY="sk-1234" \
+ --name schema-seed \
+ --add-host=host.docker.internal:host-gateway \
+ -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
+ litellm-docker-database:ci \
+ --config /app/config.yaml \
+ --port 4000 \
+ --use_prisma_db_push
+ - run:
+ name: Wait for schema seed to complete
+ command: dockerize -wait http://localhost:4001 -timeout 5m
+ - run:
+ name: Stop schema seed container
+ command: docker stop schema-seed && docker rm schema-seed
+ - run:
+ name: Run Docker container with bad schema and disabled updates
command: |
docker run -d \
-p 4000:4000 \
- -e DATABASE_URL=$PROXY_DATABASE_URL \
+ -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
-e DISABLE_SCHEMA_UPDATE="True" \
+ --name my-app \
+ --add-host=host.docker.internal:host-gateway \
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
-v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \
- --name my-app \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000
- - run:
- name: Install curl and dockerize
- command: |
- sudo apt-get update
- sudo apt-get install -y curl
- sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
- sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
- sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
-
- run:
name: Wait for container to be ready
command: dockerize -wait http://localhost:4000 -timeout 1m
@@ -2330,23 +1640,19 @@ jobs:
- run:
name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion)
command: |
- python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
- no_output_timeout: 120m
+ python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
+ no_output_timeout: 15m
build_and_test:
machine:
image: ubuntu-2204:2023.10.1
- resource_class: xlarge
+ resource_class: large
working_directory: ~/project
steps:
- checkout
+ - attach_workspace:
+ at: ~/project
- setup_google_dns
- - run:
- name: Install Docker CLI (In case it's not already installed)
- command: |
- curl -fsSL https://get.docker.com | sh
- sudo usermod -aG docker $USER
- docker version
- run:
name: Install Python 3.9
command: |
@@ -2372,10 +1678,9 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
- pip install "google-cloud-aiplatform==1.43.0"
+ pip install "google-cloud-aiplatform==1.133.0"
pip install pyarrow
- pip install "boto3==1.36.0"
- pip install "aioboto3==13.4.0"
+ pip install "boto3==1.42.80"
pip install langchain
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
@@ -2383,9 +1688,9 @@ jobs:
pip install prisma
pip install fastapi
pip install jsonschema
- pip install "httpx==0.24.1"
- pip install "gunicorn==21.2.0"
- pip install "anyio==3.7.1"
+ pip install "httpx==0.28.1"
+ pip install "gunicorn==23.0.0"
+ pip install "anyio==4.8.0"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
@@ -2412,8 +1717,10 @@ jobs:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
- name: Build Docker image
- command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
+ name: Load Docker Database Image
+ command: |
+ zstd -d litellm-docker-database.tar.zst --stdout | docker load
+ docker tag litellm-docker-database:ci my-app:latest
- run:
name: Run Docker container
command: |
@@ -2468,8 +1775,8 @@ jobs:
command: |
pwd
ls
- python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
- no_output_timeout: 120m
+ python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
+ no_output_timeout: 15m
# Store test results
- store_test_results:
@@ -2477,7 +1784,7 @@ jobs:
e2e_openai_endpoints:
machine:
image: ubuntu-2204:2023.10.1
- resource_class: xlarge
+ resource_class: large
working_directory: ~/project
steps:
- checkout
@@ -2514,10 +1821,9 @@ jobs:
pip install "mypy==1.18.2"
pip install "jsonlines==4.0.0"
pip install "google-generativeai==0.3.2"
- pip install "google-cloud-aiplatform==1.43.0"
+ pip install "google-cloud-aiplatform==1.133.0"
pip install pyarrow
- pip install "boto3==1.36.0"
- pip install "aioboto3==13.4.0"
+ pip install "boto3==1.42.80"
pip install langchain
pip install "langchain_mcp_adapters==0.0.5"
pip install "langfuse>=2.0.0"
@@ -2526,9 +1832,9 @@ jobs:
pip install prisma
pip install fastapi
pip install jsonschema
- pip install "httpx==0.24.1"
- pip install "gunicorn==21.2.0"
- pip install "anyio==3.7.1"
+ pip install "httpx==0.28.1"
+ pip install "gunicorn==23.0.0"
+ pip install "anyio==4.8.0"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
@@ -2558,7 +1864,7 @@ jobs:
- run:
name: Load Docker Database Image
command: |
- gunzip -c litellm-docker-database.tar.gz | docker load
+ zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
@@ -2616,7 +1922,7 @@ jobs:
pwd
ls
python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
+ no_output_timeout: 15m
# Store test results
- store_test_results:
@@ -2624,7 +1930,7 @@ jobs:
proxy_logging_guardrails_model_info_tests:
machine:
image: ubuntu-2204:2023.10.1
- resource_class: xlarge
+ resource_class: large
working_directory: ~/project
steps:
- checkout
@@ -2660,10 +1966,9 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
- pip install "google-cloud-aiplatform==1.43.0"
+ pip install "google-cloud-aiplatform==1.133.0"
pip install pyarrow
- pip install "boto3==1.36.0"
- pip install "aioboto3==13.4.0"
+ pip install "boto3==1.42.80"
pip install langchain
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
@@ -2671,9 +1976,9 @@ jobs:
pip install prisma
pip install fastapi
pip install jsonschema
- pip install "httpx==0.24.1"
- pip install "gunicorn==21.2.0"
- pip install "anyio==3.7.1"
+ pip install "httpx==0.28.1"
+ pip install "gunicorn==23.0.0"
+ pip install "anyio==4.8.0"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
@@ -2702,7 +2007,7 @@ jobs:
- run:
name: Load Docker Database Image
command: |
- gunzip -c litellm-docker-database.tar.gz | docker load
+ zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
@@ -2719,9 +2024,6 @@ jobs:
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e OTEL_EXPORTER="in_memory" \
- -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \
- -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \
- -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
@@ -2729,7 +2031,6 @@ jobs:
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e AWS_REGION_NAME=$AWS_REGION_NAME \
- -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \
-e COHERE_API_KEY=$COHERE_API_KEY \
-e GCS_FLUSH_INTERVAL="1" \
--add-host host.docker.internal:host-gateway \
@@ -2760,9 +2061,8 @@ jobs:
command: |
pwd
ls
- python -m pytest -vv tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5
- no_output_timeout:
- 120m
+ python -m pytest -v tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5
+ no_output_timeout: 15m
# Clean up first container
- run:
name: Stop and remove first container
@@ -2804,8 +2104,8 @@ jobs:
- run:
name: Run second round of tests
command: |
- python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
- no_output_timeout: 120m
+ python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
+ no_output_timeout: 15m
# Store test results
- store_test_results:
@@ -2813,7 +2113,7 @@ jobs:
proxy_spend_accuracy_tests:
machine:
image: ubuntu-2204:2023.10.1
- resource_class: xlarge
+ resource_class: large
working_directory: ~/project
steps:
- checkout
@@ -2867,7 +2167,7 @@ jobs:
- run:
name: Load Docker Database Image
command: |
- gunzip -c litellm-docker-database.tar.gz | docker load
+ zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
@@ -2914,8 +2214,7 @@ jobs:
pwd
ls
python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5
- no_output_timeout:
- 120m
+ no_output_timeout: 15m
# Clean up first container
- run:
name: Stop and remove first container
@@ -2926,7 +2225,7 @@ jobs:
proxy_multi_instance_tests:
machine:
image: ubuntu-2204:2023.10.1
- resource_class: xlarge
+ resource_class: large
working_directory: ~/project
steps:
- checkout
@@ -2984,7 +2283,7 @@ jobs:
- run:
name: Load Docker Database Image
command: |
- gunzip -c litellm-docker-database.tar.gz | docker load
+ zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container 1
@@ -3054,8 +2353,7 @@ jobs:
pwd
ls
python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5
- no_output_timeout:
- 120m
+ no_output_timeout: 15m
# Clean up first container
# Store test results
- store_test_results:
@@ -3064,7 +2362,7 @@ jobs:
proxy_store_model_in_db_tests:
machine:
image: ubuntu-2204:2023.10.1
- resource_class: xlarge
+ resource_class: large
working_directory: ~/project
steps:
- checkout
@@ -3124,7 +2422,7 @@ jobs:
- run:
name: Load Docker Database Image
command: |
- gunzip -c litellm-docker-database.tar.gz | docker load
+ zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
@@ -3165,7 +2463,7 @@ jobs:
pwd
ls
python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
+ no_output_timeout: 15m
- run:
name: Stop and remove containers
command: |
@@ -3181,7 +2479,7 @@ jobs:
# Change from docker to machine executor
machine:
image: ubuntu-2204:2023.10.1
- resource_class: xlarge
+ resource_class: large
working_directory: ~/project
steps:
- checkout
@@ -3201,20 +2499,27 @@ jobs:
- run:
name: Install Dependencies
command: |
- pip install "pytest==7.3.1"
- pip install "pytest-asyncio==0.21.1"
- pip install aiohttp
python -m pip install --upgrade pip
- pip install "pytest==7.3.1"
- pip install "pytest-retry==1.6.3"
- pip install "pytest-mock==3.12.0"
- pip install "pytest-asyncio==0.21.1"
- pip install "mypy==1.18.2"
- pip install apscheduler
+ pip install "pytest==7.3.1" "pytest-asyncio==0.21.1" "pytest-retry==1.6.3" \
+ "pytest-mock==3.12.0" "mypy==1.18.2" aiohttp apscheduler
- run:
name: Build Docker image
command: |
docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip .
+ - run:
+ name: Start PostgreSQL Database
+ command: |
+ docker run -d \
+ --name postgres-db \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_PASSWORD=postgres \
+ -e POSTGRES_DB=circle_test \
+ -p 5432:5432 \
+ postgres:14
+ - run:
+ name: Wait for PostgreSQL to be ready
+ command: |
+ timeout 60s bash -c 'until docker exec postgres-db pg_isready -U postgres -d circle_test; do sleep 2; done'
- run:
name: Run Docker container
# intentionally give bad redis credentials here
@@ -3222,7 +2527,7 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
- -e DATABASE_URL=$PROXY_DATABASE_URL \
+ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
@@ -3230,18 +2535,15 @@ jobs:
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e OTEL_EXPORTER="in_memory" \
- -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \
- -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \
- -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e AWS_REGION_NAME=$AWS_REGION_NAME \
- -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \
-e COHERE_API_KEY=$COHERE_API_KEY \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e GCS_FLUSH_INTERVAL="1" \
+ --add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \
my-app:latest \
@@ -3267,28 +2569,24 @@ jobs:
name: Run tests
command: |
python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
- no_output_timeout:
- 120m
+ no_output_timeout: 15m
# Clean up first container
- run:
name: Stop and remove first container
command: |
- docker stop my-app
- docker rm my-app
+ docker stop my-app || true
+ docker rm my-app || true
+ docker stop postgres-db || true
+ docker rm postgres-db || true
+ when: always
proxy_pass_through_endpoint_tests:
machine:
image: ubuntu-2204:2023.10.1
- resource_class: xlarge
+ resource_class: large
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- - run:
- name: Install Docker CLI (In case it's not already installed)
- command: |
- curl -fsSL https://get.docker.com | sh
- sudo usermod -aG docker $USER
- docker version
- run:
name: Install Python 3.10
command: |
@@ -3309,16 +2607,16 @@ jobs:
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
- pip install "google-cloud-aiplatform==1.43.0"
+ pip install "google-cloud-aiplatform==1.133.0"
pip install aiohttp
pip install "openai==1.100.1"
pip install "assemblyai==0.37.0"
python -m pip install --upgrade pip
- pip install "pydantic==2.10.2"
+ pip install "pydantic==2.12.5"
pip install "pytest==7.3.1"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
- pip install "boto3==1.36.0"
+ pip install "boto3==1.42.80"
pip install "mypy==1.18.2"
pip install pyarrow
pip install numpydoc
@@ -3326,11 +2624,11 @@ jobs:
pip install fastapi
pip install jsonschema
pip install "httpx==0.27.0"
- pip install "anyio==3.7.1"
+ pip install "anyio==4.8.0"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "google-cloud-aiplatform==1.59.0"
- pip install "anthropic==0.52.0"
+ pip install "anthropic==0.54.0"
pip install "langchain_mcp_adapters==0.0.5"
pip install "langchain_openai==0.2.1"
pip install "langgraph==0.3.18"
@@ -3360,7 +2658,7 @@ jobs:
- run:
name: Load Docker Database Image
command: |
- gunzip -c litellm-docker-database.tar.gz | docker load
+ zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
@@ -3379,6 +2677,7 @@ jobs:
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
+ -e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
@@ -3456,8 +2755,8 @@ jobs:
conda activate myenv
pwd
ls
- python -m pytest -vv tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
+ python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5
+ no_output_timeout: 15m
# Store test results
- store_test_results:
@@ -3466,7 +2765,7 @@ jobs:
proxy_e2e_anthropic_messages_tests:
machine:
image: ubuntu-2204:2023.10.1
- resource_class: xlarge
+ resource_class: large
working_directory: ~/project
steps:
- checkout
@@ -3496,7 +2795,7 @@ jobs:
conda activate myenv
pip install "pytest==7.3.1"
pip install "pytest-asyncio==0.21.1"
- pip install "boto3==1.36.0"
+ pip install "boto3==1.42.80"
pip install "httpx==0.27.0"
pip install "claude-agent-sdk"
pip install -r requirements.txt
@@ -3524,7 +2823,7 @@ jobs:
- run:
name: Load Docker Database Image
command: |
- gunzip -c litellm-docker-database.tar.gz | docker load
+ zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container with test config
@@ -3533,9 +2832,11 @@ jobs:
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e LITELLM_MASTER_KEY="sk-1234" \
+ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e AWS_REGION_NAME="us-east-1" \
+ -e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \
@@ -3561,12 +2862,120 @@ jobs:
pwd
ls
python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
+ no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
+ proxy_e2e_azure_batches_tests:
+ machine:
+ image: ubuntu-2204:2023.10.1
+ resource_class: large
+ working_directory: ~/project
+ steps:
+ - checkout
+ - setup_google_dns
+ - run:
+ name: Install Docker CLI
+ command: |
+ curl -fsSL https://get.docker.com | sh
+ sudo usermod -aG docker $USER
+ docker version
+ - run:
+ name: Install Python 3.12
+ command: |
+ curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
+ bash miniconda.sh -b -p $HOME/miniconda
+ export PATH="$HOME/miniconda/bin:$PATH"
+ conda init bash
+ source ~/.bashrc
+ conda create -n myenv python=3.12 -y
+ conda activate myenv
+ python --version
+ - run:
+ name: Install Poetry
+ command: |
+ export PATH="$HOME/miniconda/bin:$PATH"
+ source $HOME/miniconda/etc/profile.d/conda.sh
+ conda activate myenv
+ pip install poetry
+ - run:
+ name: Install dockerize
+ command: |
+ wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+ sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+ rm dockerize-linux-amd64-v0.6.1.tar.gz
+ - run:
+ name: Start PostgreSQL Database
+ command: |
+ docker run -d \
+ --name postgres-db \
+ -e POSTGRES_USER=llmproxy \
+ -e POSTGRES_PASSWORD=dbpassword9090 \
+ -e POSTGRES_DB=litellm \
+ -p 5432:5432 \
+ postgres:15
+ - run:
+ name: Wait for PostgreSQL to be ready
+ command: dockerize -wait tcp://localhost:5432 -timeout 1m
+ - run:
+ name: Install system dependencies
+ command: |
+ sudo apt-get update -y
+ sudo apt-get install -y libpq-dev
+ - run:
+ name: Install Dependencies
+ command: |
+ export PATH="$HOME/miniconda/bin:$PATH"
+ source $HOME/miniconda/etc/profile.d/conda.sh
+ conda activate myenv
+ poetry config virtualenvs.in-project true
+ poetry install --with dev,proxy-dev --extras "proxy"
+ poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity
+ - run:
+ name: Setup litellm-enterprise
+ command: |
+ export PATH="$HOME/miniconda/bin:$PATH"
+ source $HOME/miniconda/etc/profile.d/conda.sh
+ conda activate myenv
+ poetry run pip install --force-reinstall --no-deps -e enterprise/
+ - run:
+ name: Generate Prisma client
+ command: |
+ export PATH="$HOME/miniconda/bin:$PATH"
+ source $HOME/miniconda/etc/profile.d/conda.sh
+ conda activate myenv
+ poetry run prisma generate --schema litellm/proxy/schema.prisma
+ - run:
+ name: Run Prisma migrations
+ command: |
+ export PATH="$HOME/miniconda/bin:$PATH"
+ source $HOME/miniconda/etc/profile.d/conda.sh
+ conda activate myenv
+ export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+ cd litellm/proxy
+ poetry run prisma migrate deploy --schema schema.prisma
+ cd ../..
+ - run:
+ name: Run Azure Batch E2E Tests
+ command: |
+ export PATH="$HOME/miniconda/bin:$PATH"
+ source $HOME/miniconda/etc/profile.d/conda.sh
+ conda activate myenv
+ export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+ export USE_LOCAL_LITELLM=true
+ export USE_MOCK_MODELS=true
+ export USE_STATE_TRACKER=true
+ export LITELLM_LOG=DEBUG
+ poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
+ -vv -s -k "test_e2e_managed_batch" \
+ --tb=short \
+ --maxfail=3 \
+ --durations=10 \
+ --junitxml=test-results/junit.xml
+ no_output_timeout: 15m
+
upload-coverage:
docker:
- image: cimg/python:3.9
@@ -3588,98 +2997,11 @@ jobs:
python -m venv venv
. venv/bin/activate
pip install coverage
- coverage combine llm_translation_coverage realtime_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
+ coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage
coverage xml
- codecov/upload:
file: ./coverage.xml
- publish_to_pypi:
- docker:
- - image: cimg/python:3.8
- working_directory: ~/project
-
- environment:
- TWINE_USERNAME: __token__
-
- steps:
- - checkout
-
- - run:
- name: Copy model_prices_and_context_window File to model_prices_and_context_window_backup
- command: |
- cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
-
- - run:
- name: Checkout code
- command: git checkout $CIRCLE_SHA1
-
- # Check if setup.py is modified and publish to PyPI
- - run:
- name: PyPI publish
- command: |
- echo "Install TOML package."
- python -m pip install toml
- VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
- PACKAGE_NAME=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['name'])")
- if ! pip show -v $PACKAGE_NAME | grep -q "Version: ${VERSION}"; then
- echo "pyproject.toml modified"
- echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc
- python -m pip install --upgrade pip
- pip install build
- pip install wheel
- pip install --upgrade twine setuptools
- rm -rf build dist
-
- echo "Building package"
- python -m build
-
- echo "Twine upload to dist"
- echo "Contents of dist directory:"
- ls dist/
- twine upload --verbose dist/*
- else
- echo "Version ${VERSION} of package is already published on PyPI."
-
- # Check if corresponding Docker nightly image exists
- NIGHTLY_TAG="v${VERSION}-nightly"
- echo "Checking for Docker nightly image: litellm/litellm:${NIGHTLY_TAG}"
-
- # Check Docker Hub for the nightly image
- if curl -s "https://hub.docker.com/v2/repositories/litellm/litellm/tags/${NIGHTLY_TAG}" | grep -q "name"; then
- echo "Docker nightly image ${NIGHTLY_TAG} exists. This release was already completed successfully."
- echo "Skipping PyPI publish and continuing to ensure Docker images are up to date."
- circleci step halt
- else
- echo "ERROR: PyPI package ${VERSION} exists but Docker nightly image ${NIGHTLY_TAG} does not exist!"
- echo "This indicates an incomplete release. Please investigate."
- exit 1
- fi
- fi
- - run:
- name: Trigger Github Action for new Docker Container + Trigger Load Testing
- command: |
- echo "Install TOML package."
- python3 -m pip install toml
- VERSION=$(python3 -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
- echo "LiteLLM Version ${VERSION}"
-
- # Determine which branch to use for Docker build
- if [[ "$CIRCLE_BRANCH" =~ ^litellm_release_day_.* ]]; then
- BUILD_BRANCH="$CIRCLE_BRANCH"
- echo "Using release branch: $BUILD_BRANCH"
- else
- BUILD_BRANCH="main"
- echo "Using default branch: $BUILD_BRANCH"
- fi
-
- curl -X POST \
- -H "Accept: application/vnd.github.v3+json" \
- -H "Authorization: Bearer $GITHUB_TOKEN" \
- "https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \
- -d "{\"ref\":\"${BUILD_BRANCH}\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}"
- echo "triggering load testing server for version ${VERSION} and commit ${CIRCLE_SHA1}"
- curl -X POST "https://proxyloadtester-production.up.railway.app/start/load/test?version=${VERSION}&commit_hash=${CIRCLE_SHA1}&release_type=nightly"
-
publish_proxy_extras:
docker:
- image: cimg/python:3.8
@@ -3756,7 +3078,6 @@ jobs:
- run:
name: Get new version
command: |
- cd litellm-proxy-extras
NEW_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
echo "export NEW_VERSION=$NEW_VERSION" >> $BASH_ENV
@@ -3765,7 +3086,7 @@ jobs:
command: |
cd ~/project
# Check pyproject.toml
- CURRENT_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['dependencies']['litellm-proxy-extras'].split('\"')[1])")
+ CURRENT_VERSION=$(python -c "import toml; dep = toml.load('pyproject.toml')['tool']['poetry']['dependencies']['litellm-proxy-extras']; print(dep['version'] if isinstance(dep, dict) else dep)")
if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then
echo "Error: Version in pyproject.toml ($CURRENT_VERSION) doesn't match new version ($NEW_VERSION)"
exit 1
@@ -3781,7 +3102,6 @@ jobs:
- run:
name: Publish to PyPI
command: |
- cd litellm-proxy-extras
echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc
python -m pip install --upgrade pip build twine setuptools wheel
rm -rf build dist
@@ -3789,83 +3109,87 @@ jobs:
twine upload --verbose dist/*
ui_build:
- machine:
- image: ubuntu-2204:2023.10.1
- resource_class: xlarge
+ docker:
+ - image: cimg/node:20.19
+ auth:
+ username: ${DOCKERHUB_USERNAME}
+ password: ${DOCKERHUB_PASSWORD}
+ resource_class: medium+
working_directory: ~/project
steps:
- checkout
- setup_google_dns
+ - restore_cache:
+ keys:
+ - ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+ - ui-build-deps-v1-
+ - restore_cache:
+ keys:
+ - ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+ - ui-nextjs-cache-v1-
+ - run:
+ name: Install dependencies
+ command: |
+ cd ui/litellm-dashboard
+ npm ci
+ - save_cache:
+ key: ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+ paths:
+ - ui/litellm-dashboard/node_modules
- run:
name: Build UI
command: |
- # Set up nvm
- export NVM_DIR="/opt/circleci/.nvm"
- source "$NVM_DIR/nvm.sh"
- source "$NVM_DIR/bash_completion"
-
- # Install and use Node version
- nvm install v20
- nvm use v20
-
cd ui/litellm-dashboard
-
- # Remove node_modules and package-lock to ensure clean install (fixes dependency resolution issues)
- rm -rf node_modules package-lock.json
-
- # Install dependencies first
- npm install
-
- # Now source the build script
source ./build_ui.sh
+ - save_cache:
+ key: ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+ paths:
+ - ui/litellm-dashboard/.next/cache
- persist_to_workspace:
root: .
paths:
- litellm/proxy/_experimental/out
ui_unit_tests:
- machine:
- image: ubuntu-2204:2023.10.1
+ docker:
+ - image: cimg/node:20.19
+ auth:
+ username: ${DOCKERHUB_USERNAME}
+ password: ${DOCKERHUB_PASSWORD}
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
+ - restore_cache:
+ keys:
+ - ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+ - ui-unit-deps-v1-
+ - run:
+ name: Install dependencies
+ command: |
+ cd ui/litellm-dashboard
+ npm ci
+ - save_cache:
+ key: ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+ paths:
+ - ui/litellm-dashboard/node_modules
- run:
name: Run UI unit tests (Vitest)
command: |
- # Use Node 20 (several deps require >=20)
- export NVM_DIR="/opt/circleci/.nvm"
- source "$NVM_DIR/nvm.sh"
- nvm install 20
- nvm use 20
-
cd ui/litellm-dashboard
- # Remove node_modules and package-lock to ensure clean install (fixes optional deps issue)
- rm -rf node_modules package-lock.json
- npm install
- # CI run, with both LCOV (Codecov) and HTML (artifact you can click)
- CI=true npm run test -- --run --coverage \
- --coverage.provider=v8 \
- --coverage.reporter=lcov \
- --coverage.reporter=html \
- --coverage.reportsDirectory=coverage/html
+ CI=true npm run test -- --run \
+ --pool forks --poolOptions.forks.maxForks=8
build_docker_database_image:
machine:
- image: ubuntu-2204:2023.10.1
- resource_class: xlarge
+ image: ubuntu-2204:2024.04.1
+ resource_class: large
working_directory: ~/project
steps:
- checkout
- - run:
- name: Upgrade Docker
- command: |
- curl -fsSL https://get.docker.com | sh
- docker version
-
- run:
name: Build Docker image
command: |
@@ -3876,18 +3200,21 @@ jobs:
- run:
name: Save Docker image to workspace root
command: |
- docker save litellm-docker-database:ci | gzip > litellm-docker-database.tar.gz
+ docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst
- persist_to_workspace:
root: .
paths:
- - litellm-docker-database.tar.gz
+ - litellm-docker-database.tar.zst
e2e_ui_testing:
machine:
image: ubuntu-2204:2023.10.1
- resource_class: xlarge
+ resource_class: large
working_directory: ~/project
+ parameters:
+ browser:
+ type: string
steps:
- checkout
- setup_google_dns
@@ -3896,7 +3223,7 @@ jobs:
- run:
name: Load Docker Database Image
command: |
- gunzip -c litellm-docker-database.tar.gz | docker load
+ zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
name: Install Dependencies
@@ -3906,31 +3233,9 @@ jobs:
name: Install Playwright Browsers
command: |
npx playwright install
- - run:
- name: Install Neon CLI
- command: |
- npm i -g neonctl
- - run:
- name: Create Neon branch
- command: |
- export EXPIRES_AT=$(date -u -d "+3 hours" +"%Y-%m-%dT%H:%M:%SZ")
- echo "Expires at: $EXPIRES_AT"
- neon branches create \
- --project-id $NEON_PROJECT_ID \
- --name preview/commit-${CIRCLE_SHA1:0:7} \
- --expires-at $EXPIRES_AT \
- --parent br-fancy-paper-ad1olsb3 \
- --api-key $NEON_API_KEY || true
- run:
name: Run Docker container
command: |
- E2E_UI_TEST_DATABASE_URL=$(neon connection-string \
- --project-id $NEON_PROJECT_ID \
- --api-key $NEON_API_KEY \
- --branch preview/commit-${CIRCLE_SHA1:0:7} \
- --database-name yuneng-trial-db \
- --role neondb_owner)
- echo $E2E_UI_TEST_DATABASE_URL
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \
@@ -3939,7 +3244,7 @@ jobs:
-e UI_USERNAME="admin" \
-e UI_PASSWORD="gm" \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
- --name litellm-docker-database \
+ --name litellm-docker-database-<< parameters.browser >> \
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
@@ -3955,7 +3260,7 @@ jobs:
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start outputting logs
- command: docker logs -f litellm-docker-database
+ command: docker logs -f litellm-docker-database-<< parameters.browser >>
background: true
- run:
name: Wait for app to be ready
@@ -3964,10 +3269,11 @@ jobs:
name: Run Playwright Tests
command: |
npx playwright test \
+ --project << parameters.browser >> \
--config ui/litellm-dashboard/e2e_tests/playwright.config.ts \
--reporter=html \
--output=test-results
- no_output_timeout: 120m
+ no_output_timeout: 15m
- store_artifacts:
path: test-results
destination: playwright-results
@@ -3976,36 +3282,75 @@ jobs:
path: playwright-report
destination: playwright-report
- test_nonroot_image:
+ prisma_schema_sync:
machine:
image: ubuntu-2204:2023.10.1
- resource_class: xlarge
+ resource_class: medium
working_directory: ~/project
steps:
- checkout
- setup_google_dns
+ - attach_workspace:
+ at: ~/project
- run:
- name: Build Docker image
+ name: Install dockerize
command: |
- docker build -t non_root_image:latest . -f ./docker/Dockerfile.non_root
+ sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+ sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+ sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
- name: Install Container Structure Test
+ name: Start PostgreSQL Database
command: |
- curl -LO https://github.com/GoogleContainerTools/container-structure-test/releases/download/v1.19.3/container-structure-test-linux-amd64
- chmod +x container-structure-test-linux-amd64
- sudo mv container-structure-test-linux-amd64 /usr/local/bin/container-structure-test
+ docker run -d \
+ --name postgres-db \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_PASSWORD=postgres \
+ -e POSTGRES_DB=litellm_schema_sync \
+ -p 5432:5432 \
+ postgres:14
- run:
- name: Run Container Structure Test
+ name: Wait for PostgreSQL to be ready
+ command: dockerize -wait tcp://localhost:5432 -timeout 1m
+ - run:
+ name: Load Docker Database Image
command: |
- container-structure-test test --image non_root_image:latest --config docker/tests/nonroot.yaml
+ zstd -d litellm-docker-database.tar.zst --stdout | docker load
+ docker images | grep litellm-docker-database
+ - run:
+ name: Run schema sync via prisma db push
+ command: |
+ docker run -d \
+ -p 4000:4000 \
+ -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_schema_sync" \
+ -e LITELLM_MASTER_KEY="sk-1234" \
+ --name schema-sync \
+ --add-host=host.docker.internal:host-gateway \
+ -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
+ litellm-docker-database:ci \
+ --config /app/config.yaml \
+ --port 4000 \
+ --use_prisma_db_push
+ - run:
+ name: Start outputting logs
+ command: docker logs -f schema-sync
+ background: true
+ - run:
+ name: Wait for proxy to be ready (schema sync complete)
+ command: dockerize -wait http://localhost:4000 -timeout 5m
+ - run:
+ name: Stop schema sync container
+ command: docker stop schema-sync
+
test_bad_database_url:
machine:
image: ubuntu-2204:2023.10.1
- resource_class: xlarge
+ resource_class: medium
working_directory: ~/project
steps:
- checkout
+ - attach_workspace:
+ at: ~/project
- setup_google_dns
- run:
name: Install dockerize
@@ -4027,9 +3372,10 @@ jobs:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
- name: Build Docker image
+ name: Load Docker Database Image
command: |
- docker build -t myapp . -f ./docker/Dockerfile.non_root
+ zstd -d litellm-docker-database.tar.zst --stdout | docker load
+ docker tag litellm-docker-database:ci myapp:latest
- run:
name: Run Docker container with bad DATABASE_URL
command: |
@@ -4046,7 +3392,8 @@ jobs:
name: Check for expected error
command: |
if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \
- grep -q "ERROR: Application startup failed. Exiting." docker_output.log; then
+ (grep -q "Database setup failed after multiple retries" docker_output.log || \
+ grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then
echo "Expected error found. Test passed."
else
echo "Expected error not found. Test failed."
@@ -4070,6 +3417,12 @@ workflows:
only:
- main
- /litellm_.*/
+ - semgrep:
+ filters:
+ branches:
+ only:
+ - main
+ - /litellm_.*/
- local_testing_part1:
filters:
branches:
@@ -4088,34 +3441,14 @@ workflows:
only:
- main
- /litellm_.*/
- - caching_unit_tests:
- filters:
- branches:
- only:
- main
- /litellm_.*/
- - litellm_proxy_unit_testing_key_generation:
- filters:
- branches:
- only:
- main
- /litellm_.*/
- - litellm_proxy_unit_testing_part1:
- filters:
- branches:
- only:
- main
- /litellm_.*/
- - litellm_proxy_unit_testing_part2:
- filters:
- branches:
- only:
- main
- /litellm_.*/
- - litellm_security_tests:
- filters:
- branches:
- only:
- main
- /litellm_.*/
- litellm_assistants_api_testing:
@@ -4168,17 +3501,43 @@ workflows:
only:
- main
- /litellm_.*/
- - e2e_ui_testing:
- context: e2e_ui_tests
+ - prisma_schema_sync:
requires:
- - ui_build
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
+ # - e2e_ui_testing:
+ # name: e2e_ui_testing_chromium
+ # browser: chromium
+ # context: e2e_ui_tests
+ # requires:
+ # - ui_build
+ # - build_docker_database_image
+ # - prisma_schema_sync
+ # filters:
+ # branches:
+ # only:
+ # - main
+ # - /litellm_.*/
+ # - e2e_ui_testing:
+ # name: e2e_ui_testing_firefox
+ # browser: firefox
+ # context: e2e_ui_tests
+ # requires:
+ # - ui_build
+ # - build_docker_database_image
+ # - prisma_schema_sync
+ # filters:
+ # branches:
+ # only:
+ # - main
+ # - /litellm_.*/
- build_and_test:
+ requires:
+ - build_docker_database_image
filters:
branches:
only:
@@ -4246,6 +3605,12 @@ workflows:
only:
- main
- /litellm_.*/
+ - proxy_e2e_azure_batches_tests:
+ filters:
+ branches:
+ only:
+ - main
+ - /litellm_.*/
- llm_translation_testing:
filters:
branches:
@@ -4264,6 +3629,12 @@ workflows:
only:
- main
- /litellm_.*/
+ - agent_testing:
+ filters:
+ branches:
+ only:
+ - main
+ - /litellm_.*/
- guardrails_testing:
filters:
branches:
@@ -4300,34 +3671,26 @@ workflows:
only:
- main
- /litellm_.*/
- - litellm_mapped_tests_proxy:
+ - litellm_mapped_tests_proxy_part1:
filters:
branches:
only:
- main
- /litellm_.*/
- - litellm_mapped_tests_llms:
+ - litellm_mapped_tests_proxy_part2:
filters:
branches:
only:
- main
- /litellm_.*/
- - litellm_mapped_tests_core:
- filters:
- branches:
- only:
- main
- /litellm_.*/
- - litellm_mapped_tests_integrations:
- filters:
- branches:
- only:
- main
- /litellm_.*/
- - litellm_mapped_tests_litellm_core_utils:
- filters:
- branches:
- only:
+ - main
+ - /litellm_.*/
+ - main
+ - /litellm_.*/
- main
- /litellm_.*/
- batches_testing:
@@ -4368,19 +3731,15 @@ workflows:
- /litellm_.*/
- upload-coverage:
requires:
- - llm_translation_testing
- realtime_translation_testing
- mcp_testing
+ - agent_testing
- google_generate_content_endpoint_testing
- guardrails_testing
- - llm_responses_api_testing
- ocr_testing
- search_testing
- - litellm_mapped_tests_proxy
- - litellm_mapped_tests_llms
- - litellm_mapped_tests_core
- - litellm_mapped_tests_integrations
- - litellm_mapped_tests_litellm_core_utils
+ - litellm_mapped_tests_proxy_part1
+ - litellm_mapped_tests_proxy_part2
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
@@ -4388,18 +3747,10 @@ workflows:
- image_gen_testing
- logging_testing
- audio_testing
- - litellm_router_testing
- - litellm_router_unit_testing
- - caching_unit_tests
- - litellm_proxy_unit_testing_key_generation
- - litellm_proxy_unit_testing_part1
- - litellm_proxy_unit_testing_part2
- - litellm_security_tests
- langfuse_logging_unit_tests
- local_testing_part1
- local_testing_part2
- litellm_assistants_api_testing
- - auth_ui_unit_tests
- db_migration_disable_update_check:
requires:
- build_docker_database_image
@@ -4421,12 +3772,16 @@ workflows:
- main
- /litellm_.*/
- helm_chart_testing:
+ requires:
+ - build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- test_bad_database_url:
+ requires:
+ - build_docker_database_image
filters:
branches:
only:
@@ -4438,53 +3793,3 @@ workflows:
only:
- main
- /litellm_release_day_.*/
- - publish_to_pypi:
- requires:
- - mypy_linting
- - local_testing_part1
- - local_testing_part2
- - build_and_test
- - e2e_openai_endpoints
- - test_bad_database_url
- - llm_translation_testing
- - realtime_translation_testing
- - mcp_testing
- - google_generate_content_endpoint_testing
- - llm_responses_api_testing
- - ocr_testing
- - search_testing
- - litellm_mapped_tests_proxy
- - litellm_mapped_tests_llms
- - litellm_mapped_tests_core
- - litellm_mapped_tests_integrations
- - litellm_mapped_tests_litellm_core_utils
- - litellm_mapped_enterprise_tests
- - batches_testing
- - litellm_utils_testing
- - pass_through_unit_testing
- - image_gen_testing
- - logging_testing
- - audio_testing
- - litellm_router_testing
- - litellm_router_unit_testing
- - caching_unit_tests
- - langfuse_logging_unit_tests
- - litellm_assistants_api_testing
- - auth_ui_unit_tests
- - db_migration_disable_update_check
- - e2e_ui_testing
- - litellm_proxy_unit_testing_key_generation
- - litellm_proxy_unit_testing_part1
- - litellm_proxy_unit_testing_part2
- - litellm_security_tests
- - installing_litellm_on_python
- - installing_litellm_on_python_3_13
- - proxy_logging_guardrails_model_info_tests
- - proxy_spend_accuracy_tests
- - proxy_multi_instance_tests
- - proxy_store_model_in_db_tests
- - proxy_build_from_pip_tests
- - proxy_pass_through_endpoint_tests
- - check_code_and_doc_quality
- - publish_proxy_extras
- - guardrails_testing
diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt
index a5ec74424fe..be12ab2d0f1 100644
--- a/.circleci/requirements.txt
+++ b/.circleci/requirements.txt
@@ -7,14 +7,15 @@ cohere
redis==5.2.1
redisvl==0.4.1
anthropic
-orjson==3.10.12 # fast /embedding responses
-pydantic==2.11.0
-google-cloud-aiplatform==1.43.0
+orjson==3.10.15 # fast /embedding responses
+pydantic==2.12.5
+google-cloud-aiplatform==1.133.0
google-cloud-iam==2.19.1
fastapi-sso==0.16.0
uvloop==0.21.0
-mcp==1.25.0 # for MCP server
+mcp==1.26.0 # for MCP server
semantic_router==0.1.10 # for auto-routing with litellm
-fastuuid==0.12.0
+fastuuid==0.14.0
responses==0.25.7 # for proxy client tests
-pytest-retry==1.6.3 # for automatic test retries
\ No newline at end of file
+pytest-retry==1.6.3 # for automatic test retries
+litellm-proxy-extras # for prisma migrations
\ No newline at end of file
diff --git a/.github/actions/helm-oci-chart-releaser/action.yml b/.github/actions/helm-oci-chart-releaser/action.yml
index 059277ed882..454c591d436 100644
--- a/.github/actions/helm-oci-chart-releaser/action.yml
+++ b/.github/actions/helm-oci-chart-releaser/action.yml
@@ -40,38 +40,55 @@ outputs:
runs:
using: composite
steps:
+ - name: Helm | Setup
+ uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
+ with:
+ version: v3.20.0
+
- name: Helm | Login
shell: bash
- run: echo ${{ inputs.registry_password }} | helm registry login -u ${{ inputs.registry_username }} --password-stdin ${{ inputs.registry }}
env:
- HELM_EXPERIMENTAL_OCI: '1'
-
+ REGISTRY_PASSWORD: ${{ inputs.registry_password }}
+ REGISTRY_USERNAME: ${{ inputs.registry_username }}
+ REGISTRY: ${{ inputs.registry }}
+ run: echo "$REGISTRY_PASSWORD" | helm registry login -u "$REGISTRY_USERNAME" --password-stdin "$REGISTRY"
+
- name: Helm | Dependency
if: inputs.update_dependencies == 'true'
shell: bash
- run: helm dependency update ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
env:
- HELM_EXPERIMENTAL_OCI: '1'
+ CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
+ run: helm dependency update "$CHART_PATH"
- name: Helm | Package
shell: bash
- run: helm package ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} --version ${{ inputs.tag }} --app-version ${{ inputs.app_version }}
env:
- HELM_EXPERIMENTAL_OCI: '1'
+ CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
+ TAG: ${{ inputs.tag }}
+ APP_VERSION: ${{ inputs.app_version }}
+ run: helm package "$CHART_PATH" --version "$TAG" --app-version "$APP_VERSION"
- name: Helm | Push
shell: bash
- run: helm push ${{ inputs.name }}-${{ inputs.tag }}.tgz oci://${{ inputs.registry }}/${{ inputs.repository }}
env:
- HELM_EXPERIMENTAL_OCI: '1'
+ NAME: ${{ inputs.name }}
+ TAG: ${{ inputs.tag }}
+ REGISTRY: ${{ inputs.registry }}
+ REPOSITORY: ${{ inputs.repository }}
+ run: helm push "${NAME}-${TAG}.tgz" "oci://${REGISTRY}/${REPOSITORY}"
- name: Helm | Logout
shell: bash
- run: helm registry logout ${{ inputs.registry }}
env:
- HELM_EXPERIMENTAL_OCI: '1'
+ REGISTRY: ${{ inputs.registry }}
+ run: helm registry logout "$REGISTRY"
- name: Helm | Output
id: output
shell: bash
- run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT
\ No newline at end of file
+ env:
+ REGISTRY: ${{ inputs.registry }}
+ REPOSITORY: ${{ inputs.repository }}
+ NAME: ${{ inputs.name }}
+ TAG: ${{ inputs.tag }}
+ run: echo "image=${REGISTRY}/${REPOSITORY}/${NAME}:${TAG}" >> $GITHUB_OUTPUT
diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml
index 58e7cfe10da..c49882a8d62 100644
--- a/.github/dependabot.yaml
+++ b/.github/dependabot.yaml
@@ -4,6 +4,9 @@ updates:
directory: "/"
schedule:
interval: "daily"
+ cooldown:
+ default-days: 7
+ semver-major-days: 14
groups:
github-actions:
patterns:
diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml
new file mode 100644
index 00000000000..f1ae30e67d7
--- /dev/null
+++ b/.github/workflows/_test-unit-base.yml
@@ -0,0 +1,96 @@
+name: _Unit Test Base (Reusable)
+
+on:
+ workflow_call:
+ inputs:
+ test-path:
+ description: "Pytest path(s) to run"
+ required: true
+ type: string
+ workers:
+ description: "Number of pytest-xdist workers"
+ required: false
+ type: number
+ default: 2
+ reruns:
+ description: "Number of reruns for flaky tests"
+ required: false
+ type: number
+ default: 2
+ timeout-minutes:
+ description: "Job timeout in minutes"
+ required: false
+ type: number
+ default: 20
+ max-failures:
+ description: "Stop after this many failures"
+ required: false
+ type: number
+ default: 10
+
+permissions:
+ contents: read
+
+jobs:
+ run:
+ name: Run tests
+ runs-on: ubuntu-latest
+ timeout-minutes: ${{ inputs.timeout-minutes }}
+
+ steps:
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+
+ - name: Install Poetry
+ run: pip install 'poetry==2.3.2'
+
+ - name: Cache Poetry dependencies
+ uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+ with:
+ path: |
+ ~/.cache/pypoetry
+ ~/.cache/pip
+ .venv
+ key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-poetry-
+
+ - name: Install dependencies
+ run: |
+ poetry config virtualenvs.in-project true
+ poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+ poetry run pip install google-genai==1.22.0 \
+ google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
+
+ - name: Setup litellm-enterprise
+ run: |
+ poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+ - name: Generate Prisma client
+ env:
+ PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+ run: |
+ poetry run pip install nodejs-wheel-binaries==24.13.1
+ poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+ - name: Run tests
+ env:
+ TEST_PATH: ${{ inputs.test-path }}
+ MAX_FAILURES: ${{ inputs.max-failures }}
+ WORKERS: ${{ inputs.workers }}
+ RERUNS: ${{ inputs.reruns }}
+ run: |
+ poetry run pytest ${TEST_PATH:?} \
+ --tb=short -vv \
+ --maxfail="${MAX_FAILURES}" \
+ -n "${WORKERS}" \
+ --reruns "${RERUNS}" \
+ --reruns-delay 1 \
+ --dist=loadscope \
+ --durations=20
diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml
new file mode 100644
index 00000000000..d53a9e8822a
--- /dev/null
+++ b/.github/workflows/_test-unit-services-base.yml
@@ -0,0 +1,164 @@
+name: _Unit Test Services Base (Reusable)
+
+on:
+ workflow_call:
+ inputs:
+ test-path:
+ description: "Pytest path(s) to run"
+ required: true
+ type: string
+ workers:
+ description: "Number of pytest-xdist workers (0 = no parallelism)"
+ required: false
+ type: number
+ default: 2
+ reruns:
+ description: "Number of reruns for flaky tests"
+ required: false
+ type: number
+ default: 2
+ timeout-minutes:
+ description: "Job timeout in minutes"
+ required: false
+ type: number
+ default: 20
+ max-failures:
+ description: "Stop after this many failures"
+ required: false
+ type: number
+ default: 10
+ enable-redis:
+ description: "Pass Redis Cloud credentials to tests via REDIS_HOST/PORT/PASSWORD env vars"
+ required: false
+ type: boolean
+ default: false
+ enable-postgres:
+ description: "Start a local Postgres service container and run Prisma migrations"
+ required: false
+ type: boolean
+ default: false
+ secrets:
+ REDIS_HOST:
+ required: false
+ REDIS_PORT:
+ required: false
+ REDIS_PASSWORD:
+ required: false
+ DATABASE_URL:
+ required: false
+ POSTGRES_USER:
+ required: false
+ POSTGRES_PASSWORD:
+ required: false
+
+permissions:
+ contents: read
+
+jobs:
+ run:
+ name: Run tests
+ runs-on: ubuntu-latest
+ timeout-minutes: ${{ inputs.timeout-minutes }}
+ # Environment is derived from the enable-* flags, not caller-controllable.
+ # This prevents callers from passing arbitrary environment names to bypass secret scoping.
+ # Note: Postgres service container always starts (GHA limitation), so any Redis job
+ # also needs Postgres secrets → uses integration-redis-postgres, not integration-redis.
+ environment: >-
+ ${{
+ inputs.enable-redis && 'integration-redis-postgres' ||
+ inputs.enable-postgres && 'integration-postgres' ||
+ ''
+ }}
+
+ services:
+ postgres:
+ image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14
+ env:
+ POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+ POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+ POSTGRES_DB: litellm_test
+ ports:
+ - 5432:5432
+ options: >-
+ --health-cmd "pg_isready"
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+
+ steps:
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+
+ - name: Install Poetry
+ run: pip install 'poetry==2.3.2'
+
+ - name: Cache Poetry dependencies
+ uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+ with:
+ path: |
+ ~/.cache/pypoetry
+ ~/.cache/pip
+ .venv
+ key: ${{ runner.os }}-poetry-services-${{ hashFiles('poetry.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-poetry-services-
+
+ - name: Install dependencies
+ run: |
+ poetry config virtualenvs.in-project true
+ poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+ poetry run pip install google-genai==1.22.0 \
+ google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
+
+ - name: Setup litellm-enterprise
+ run: |
+ poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+ - name: Generate Prisma client
+ env:
+ PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+ run: |
+ poetry run pip install nodejs-wheel-binaries==24.13.1
+ poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+ - name: Run Prisma migrations
+ if: ${{ inputs.enable-postgres }}
+ env:
+ DATABASE_URL: ${{ secrets.DATABASE_URL }}
+ run: |
+ poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
+
+ - name: Run tests
+ env:
+ TEST_PATH: ${{ inputs.test-path }}
+ MAX_FAILURES: ${{ inputs.max-failures }}
+ WORKERS: ${{ inputs.workers }}
+ RERUNS: ${{ inputs.reruns }}
+ DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }}
+ REDIS_HOST: ${{ inputs.enable-redis && secrets.REDIS_HOST || '' }}
+ REDIS_PORT: ${{ inputs.enable-redis && secrets.REDIS_PORT || '' }}
+ REDIS_PASSWORD: ${{ inputs.enable-redis && secrets.REDIS_PASSWORD || '' }}
+ run: |
+ if [ "${WORKERS}" = "0" ]; then
+ poetry run pytest ${TEST_PATH:?} \
+ --tb=short -vv \
+ --maxfail="${MAX_FAILURES}" \
+ --reruns "${RERUNS}" \
+ --reruns-delay 1 \
+ --durations=20
+ else
+ poetry run pytest ${TEST_PATH:?} \
+ --tb=short -vv \
+ --maxfail="${MAX_FAILURES}" \
+ -n "${WORKERS}" \
+ --reruns "${RERUNS}" \
+ --reruns-delay 1 \
+ --dist=loadscope \
+ --durations=20
+ fi
diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml
index e7d65242c19..60e89936219 100644
--- a/.github/workflows/auto_update_price_and_context_window.yml
+++ b/.github/workflows/auto_update_price_and_context_window.yml
@@ -2,17 +2,24 @@ name: Updates model_prices_and_context_window.json and Create Pull Request
on:
schedule:
- - cron: "0 0 * * 0" # Run every Sundays at midnight
+ - cron: "0 0 * * 0" # Run every Sundays at midnight
#- cron: "0 0 * * *" # Run daily at midnight
+permissions:
+ contents: write
+ pull-requests: write
+
jobs:
auto_update_price_and_context_window:
+ if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
- name: Install Dependencies
run: |
- pip install aiohttp
+ pip install 'aiohttp==3.13.3'
- name: Update JSON Data
run: |
python ".github/workflows/auto_update_price_and_context_window_file.py"
@@ -25,4 +32,4 @@ jobs:
--head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \
--base main
env:
- GH_TOKEN: ${{ secrets.GH_TOKEN }}
\ No newline at end of file
+ GH_TOKEN: ${{ secrets.GH_TOKEN }}
diff --git a/.github/workflows/check-schema-sync.yml b/.github/workflows/check-schema-sync.yml
new file mode 100644
index 00000000000..0e5e2804e60
--- /dev/null
+++ b/.github/workflows/check-schema-sync.yml
@@ -0,0 +1,58 @@
+name: Check Schema Sync
+
+on:
+ pull_request:
+ paths:
+ - 'schema.prisma'
+ - 'litellm/proxy/schema.prisma'
+ - 'litellm-proxy-extras/litellm_proxy_extras/schema.prisma'
+
+permissions:
+ contents: read
+
+jobs:
+ check-sync:
+ name: Verify schema.prisma copies match root
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ steps:
+ - name: Checkout PR
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
+
+ - name: Reject symlinked schema files
+ run: |
+ for f in schema.prisma litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; do
+ if [ -L "$f" ]; then
+ echo "::error file=$f::$f is a symlink, which is not allowed"
+ exit 1
+ fi
+ done
+
+ - name: Check all schemas match root
+ run: |
+ EXIT=0
+
+ diff schema.prisma litellm/proxy/schema.prisma || {
+ echo "::error file=litellm/proxy/schema.prisma::litellm/proxy/schema.prisma differs from root schema.prisma"
+ EXIT=1
+ }
+
+ diff schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma || {
+ echo "::error file=litellm-proxy-extras/litellm_proxy_extras/schema.prisma::litellm-proxy-extras/litellm_proxy_extras/schema.prisma differs from root schema.prisma"
+ EXIT=1
+ }
+
+ if [ "$EXIT" -ne 0 ]; then
+ echo ""
+ echo "Schema files are out of sync."
+ echo "The root schema.prisma is the source of truth."
+ echo ""
+ echo "To fix, run from the repo root:"
+ echo " cp schema.prisma litellm/proxy/schema.prisma"
+ echo " cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma"
+ exit 1
+ fi
+
+ echo "All schema copies are in sync with root."
diff --git a/.github/workflows/check_duplicate_issues.yml b/.github/workflows/check_duplicate_issues.yml
index 14d6964fcdb..289d78880ad 100644
--- a/.github/workflows/check_duplicate_issues.yml
+++ b/.github/workflows/check_duplicate_issues.yml
@@ -12,7 +12,7 @@ jobs:
contents: read
steps:
- name: Check for potential duplicates
- uses: wow-actions/potential-duplicates@v1
+ uses: wow-actions/potential-duplicates@4d4ea0352e0383859279938e255179dd1dbb67b5 # v1.1.0
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
label: potential-duplicate
@@ -20,10 +20,34 @@ jobs:
reaction: eyes
comment: |
**⚠️ Potential duplicate detected**
-
+
This issue appears similar to existing issue(s):
{{#issues}}
- [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
{{/issues}}
-
+
Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
+
+ - name: Checkout close script
+ if: github.event.action == 'opened'
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ sparse-checkout: .github/scripts
+ persist-credentials: false
+
+ - name: Set up Python
+ if: github.event.action == 'opened'
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.11"
+
+ - name: Auto-close if high-confidence duplicate
+ if: github.event.action == 'opened'
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ python3 .github/scripts/close_duplicate_issues.py \
+ --issue-number ${{ github.event.issue.number }} \
+ --repo ${{ github.repository }} \
+ --threshold 0.85 \
+ --close
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
new file mode 100644
index 00000000000..e86fca17c7a
--- /dev/null
+++ b/.github/workflows/codeql.yml
@@ -0,0 +1,55 @@
+name: "CodeQL"
+
+on:
+ push:
+ branches: [main]
+ pull_request:
+ branches: [main]
+ schedule:
+ # Run daily at 04:00 UTC
+ - cron: "0 4 * * *"
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+ analyze:
+ if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
+ name: Analyze (${{ matrix.language }})
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ permissions:
+ security-events: write
+ packages: read
+ actions: read
+ contents: read
+
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - language: actions
+ build-mode: none
+ - language: javascript-typescript
+ build-mode: none
+ - language: python
+ build-mode: none
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
+
+ - name: Initialize CodeQL
+ uses: github/codeql-action/init@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3
+ with:
+ languages: ${{ matrix.language }}
+ build-mode: ${{ matrix.build-mode }}
+ config-file: ./.github/codeql/codeql-config.yml
+
+ - name: Perform CodeQL Analysis
+ uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3
+ with:
+ category: "/language:${{ matrix.language }}"
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
new file mode 100644
index 00000000000..52d64addea9
--- /dev/null
+++ b/.github/workflows/codspeed.yml
@@ -0,0 +1,46 @@
+name: CodSpeed Benchmarks
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ branches:
+ - main
+ # Allow CodSpeed to trigger backtest performance analysis
+ # in order to generate initial data
+ workflow_dispatch:
+
+permissions:
+ contents: read
+ id-token: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ benchmarks:
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+
+ steps:
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+
+ - name: Install dependencies
+ run: |
+ pip install -e "."
+ pip install pytest pytest-codspeed==4.3.0
+
+ - name: Run benchmarks
+ uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1
+ with:
+ mode: simulation
+ run: pytest tests/benchmarks/ --codspeed
diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml
new file mode 100644
index 00000000000..2ae01823a96
--- /dev/null
+++ b/.github/workflows/create-release.yml
@@ -0,0 +1,93 @@
+name: Create Release
+
+on:
+ workflow_dispatch:
+ inputs:
+ tag:
+ description: "Release tag (e.g. v1.83.0-stable)"
+ required: true
+ type: string
+ commit_hash:
+ description: "Full 40-char commit SHA to target"
+ required: true
+ type: string
+
+permissions: {}
+
+jobs:
+ release:
+ name: Create Release
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ steps:
+ - name: Validate inputs
+ env:
+ TAG: ${{ inputs.tag }}
+ COMMIT_HASH: ${{ inputs.commit_hash }}
+ run: |
+ if ! echo "${COMMIT_HASH}" | grep -qE '^[0-9a-f]{40}$'; then
+ echo "::error::commit_hash must be a full 40-character commit SHA"
+ exit 1
+ fi
+ if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
+ echo "::error::tag must start with vX.Y.Z"
+ exit 1
+ fi
+
+ - name: Create release
+ env:
+ TAG: ${{ inputs.tag }}
+ COMMIT_HASH: ${{ inputs.commit_hash }}
+ uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+ with:
+ script: |
+ const tag = process.env.TAG;
+ const commitHash = process.env.COMMIT_HASH;
+
+ const cosignSection = [
+ `## Verify Docker Image Signature`,
+ ``,
+ `All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying:`,
+ ``,
+ '```bash',
+ `cosign verify \\`,
+ ` --key https://raw.githubusercontent.com/BerriAI/litellm/${tag}/cosign.pub \\`,
+ ` ghcr.io/berriai/litellm:${tag}`,
+ '```',
+ ``,
+ `Expected output:`,
+ ``,
+ '```',
+ `The following checks were performed on each of these signatures:`,
+ ` - The cosign claims were validated`,
+ ` - The signatures were verified against the specified public key`,
+ '```',
+ ``,
+ `---`,
+ ``,
+ ].join('\n');
+
+ try {
+ const response = await github.rest.repos.createRelease({
+ draft: true,
+ generate_release_notes: true,
+ target_commitish: commitHash,
+ name: tag,
+ owner: context.repo.owner,
+ prerelease: false,
+ repo: context.repo.repo,
+ tag_name: tag,
+ });
+
+ const updatedBody = cosignSection + (response.data.body ?? '');
+ await github.rest.repos.updateRelease({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ release_id: response.data.id,
+ body: updatedBody,
+ draft: false,
+ });
+ } catch (error) {
+ core.setFailed(error.message);
+ }
diff --git a/.github/workflows/create_daily_staging_branch.yml b/.github/workflows/create_daily_staging_branch.yml
index 9d0093e8b16..424d8de0a41 100644
--- a/.github/workflows/create_daily_staging_branch.yml
+++ b/.github/workflows/create_daily_staging_branch.yml
@@ -2,18 +2,22 @@ name: Create Daily Staging Branch
on:
schedule:
- - cron: '0 0,12 * * *' # Runs every 12 hours at midnight and noon UTC
- workflow_dispatch: # Allow manual trigger
+ - cron: "0 0,12 * * *" # Runs every 12 hours at midnight and noon UTC
+ workflow_dispatch: # Allow manual trigger
jobs:
create-staging-branch:
+ if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
+ permissions:
+ contents: write
steps:
- name: Checkout repository
- uses: actions/checkout@v3
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
fetch-depth: 0
+ persist-credentials: false
- name: Create daily staging branch
env:
@@ -41,3 +45,43 @@ jobs:
git push origin $BRANCH_NAME
echo "Successfully created and pushed branch: $BRANCH_NAME"
fi
+
+ create-internal-dev-branch:
+ if: github.repository == 'BerriAI/litellm'
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Create internal dev branch
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ # Configure Git user
+ git config user.name "github-actions[bot]"
+ git config user.email "github-actions[bot]@users.noreply.github.com"
+
+ # Generate branch name with MM_DD_YYYY format
+ BRANCH_NAME="litellm_internal_dev_$(date +'%m_%d_%Y')"
+ echo "Creating branch: $BRANCH_NAME"
+
+ # Fetch all branches
+ git fetch --all
+
+ # Check if the branch already exists
+ if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+ echo "Branch $BRANCH_NAME already exists. Skipping creation."
+ else
+ echo "Creating new branch: $BRANCH_NAME"
+ # Create the new branch from main
+ git checkout -b $BRANCH_NAME origin/main
+ # Push the new branch
+ git push origin $BRANCH_NAME
+ echo "Successfully created and pushed branch: $BRANCH_NAME"
+ fi
diff --git a/.github/workflows/ghcr_deploy.yml b/.github/workflows/ghcr_deploy.yml
deleted file mode 100644
index f67538a4272..00000000000
--- a/.github/workflows/ghcr_deploy.yml
+++ /dev/null
@@ -1,434 +0,0 @@
-# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
-name: Build, Publish LiteLLM Docker Image. New Release
-on:
- workflow_dispatch:
- inputs:
- tag:
- description: "The tag version you want to build"
- required: true
- release_type:
- description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'"
- type: string
- default: "latest"
- commit_hash:
- description: "Commit hash"
- required: true
-
-# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
-env:
- REGISTRY: ghcr.io
- IMAGE_NAME: ${{ github.repository }}
- CHART_NAME: litellm-helm
-
-# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
-jobs:
- # print commit hash, tag, and release type
- print:
- runs-on: ubuntu-latest
- steps:
- - run: |
- echo "Commit hash: ${{ github.event.inputs.commit_hash }}"
- echo "Tag: ${{ github.event.inputs.tag }}"
- echo "Release type: ${{ github.event.inputs.release_type }}"
- docker-hub-deploy:
- if: github.repository == 'BerriAI/litellm'
- runs-on: ubuntu-latest
- steps:
- -
- name: Checkout
- uses: actions/checkout@v4
- with:
- ref: ${{ github.event.inputs.commit_hash }}
- -
- name: Set up QEMU
- uses: docker/setup-qemu-action@v3
- -
- name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v3
- -
- name: Login to Docker Hub
- uses: docker/login-action@v3
- with:
- username: ${{ secrets.DOCKERHUB_USERNAME }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
- -
- name: Build and push
- uses: docker/build-push-action@v5
- with:
- context: .
- push: true
- tags: litellm/litellm:${{ github.event.inputs.tag || 'latest' }}
- -
- name: Build and push litellm-database image
- uses: docker/build-push-action@v5
- with:
- context: .
- push: true
- file: ./docker/Dockerfile.database
- tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }}
- -
- name: Build and push litellm-spend-logs image
- uses: docker/build-push-action@v5
- with:
- context: .
- push: true
- file: ./litellm-js/spend-logs/Dockerfile
- tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }}
- -
- name: Build and push litellm-non_root image
- uses: docker/build-push-action@v5
- with:
- context: .
- push: true
- file: ./docker/Dockerfile.non_root
- tags: litellm/litellm-non_root:${{ github.event.inputs.tag || 'latest' }}
- build-and-push-image:
- runs-on: ubuntu-latest
- # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
- permissions:
- contents: read
- packages: write
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
- with:
- ref: ${{ github.event.inputs.commit_hash }}
- # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
- - name: Log in to the Container registry
- uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
- with:
- registry: ${{ env.REGISTRY }}
- username: ${{ github.actor }}
- password: ${{ secrets.GITHUB_TOKEN }}
- # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
- - name: Extract metadata (tags, labels) for Docker
- id: meta
- uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
- with:
- images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- # Configure multi platform Docker builds
- - name: Set up QEMU
- uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
- # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
- # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
- # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
- - name: Build and push Docker image
- uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8
- with:
- context: .
- push: true
- tags: |
- ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
- ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }}
- ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
- ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }},
- ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
- labels: ${{ steps.meta.outputs.labels }}
- platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
-
- build-and-push-image-ee:
- runs-on: ubuntu-latest
- permissions:
- contents: read
- packages: write
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
- with:
- ref: ${{ github.event.inputs.commit_hash }}
-
- - name: Log in to the Container registry
- uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
- with:
- registry: ${{ env.REGISTRY }}
- username: ${{ github.actor }}
- password: ${{ secrets.GITHUB_TOKEN }}
-
- - name: Extract metadata (tags, labels) for EE Dockerfile
- id: meta-ee
- uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
- with:
- images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
- # Configure multi platform Docker builds
- - name: Set up QEMU
- uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
-
- - name: Build and push EE Docker image
- uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
- with:
- context: .
- file: Dockerfile
- push: true
- tags: |
- ${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
- ${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.release_type }}
- ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-ee:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
- ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-ee:main-stable', env.REGISTRY) || '' }}
- labels: ${{ steps.meta-ee.outputs.labels }}
- platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
-
- build-and-push-image-database:
- runs-on: ubuntu-latest
- permissions:
- contents: read
- packages: write
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
- with:
- ref: ${{ github.event.inputs.commit_hash }}
-
- - name: Log in to the Container registry
- uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
- with:
- registry: ${{ env.REGISTRY }}
- username: ${{ github.actor }}
- password: ${{ secrets.GITHUB_TOKEN }}
-
- - name: Extract metadata (tags, labels) for database Dockerfile
- id: meta-database
- uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
- with:
- images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-database
- # Configure multi platform Docker builds
- - name: Set up QEMU
- uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
-
- - name: Build and push Database Docker image
- uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
- with:
- context: .
- file: ./docker/Dockerfile.database
- push: true
- tags: |
- ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
- ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }}
- ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
- ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-stable', env.REGISTRY) || '' }}
- labels: ${{ steps.meta-database.outputs.labels }}
- platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
-
- build-and-push-image-non_root:
- runs-on: ubuntu-latest
- permissions:
- contents: read
- packages: write
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
- with:
- ref: ${{ github.event.inputs.commit_hash }}
-
- - name: Log in to the Container registry
- uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
- with:
- registry: ${{ env.REGISTRY }}
- username: ${{ github.actor }}
- password: ${{ secrets.GITHUB_TOKEN }}
-
- - name: Extract metadata (tags, labels) for non_root Dockerfile
- id: meta-non_root
- uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
- with:
- images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-non_root
- # Configure multi platform Docker builds
- - name: Set up QEMU
- uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
-
- - name: Build and push non_root Docker image
- uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
- with:
- context: .
- file: ./docker/Dockerfile.non_root
- push: true
- tags: |
- ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
- ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }}
- ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
- ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-stable', env.REGISTRY) || '' }}
- labels: ${{ steps.meta-non_root.outputs.labels }}
- platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
-
- build-and-push-image-spend-logs:
- runs-on: ubuntu-latest
- permissions:
- contents: read
- packages: write
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
- with:
- ref: ${{ github.event.inputs.commit_hash }}
-
- - name: Log in to the Container registry
- uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
- with:
- registry: ${{ env.REGISTRY }}
- username: ${{ github.actor }}
- password: ${{ secrets.GITHUB_TOKEN }}
-
- - name: Extract metadata (tags, labels) for spend-logs Dockerfile
- id: meta-spend-logs
- uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
- with:
- images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-spend_logs
- # Configure multi platform Docker builds
- - name: Set up QEMU
- uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
-
- - name: Build and push Database Docker image
- uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
- with:
- context: .
- file: ./litellm-js/spend-logs/Dockerfile
- push: true
- tags: |
- ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
- ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }}
- ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
- ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
- platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
-
- build-and-push-helm-chart:
- if: github.event.inputs.release_type != 'dev'
- needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
- runs-on: ubuntu-latest
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
- with:
- fetch-depth: 0
-
- - name: Log in to the Container registry
- uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
- with:
- registry: ${{ env.REGISTRY }}
- username: ${{ github.actor }}
- password: ${{ secrets.GITHUB_TOKEN }}
-
- - name: lowercase github.repository_owner
- run: |
- echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
-
- # Sync Helm chart version with LiteLLM release version (1-1 versioning)
- # This allows users to easily map Helm chart versions to LiteLLM versions
- # See: https://codefresh.io/docs/docs/ci-cd-guides/helm-best-practices/
- - name: Calculate chart and app versions
- id: chart_version
- shell: bash
- run: |
- INPUT_TAG="${{ github.event.inputs.tag }}"
- RELEASE_TYPE="${{ github.event.inputs.release_type }}"
-
- # Chart version = LiteLLM version without 'v' prefix (Helm semver convention)
- # v1.81.0 -> 1.81.0, v1.81.0.rc.1 -> 1.81.0.rc.1
- CHART_VERSION="${INPUT_TAG#v}"
-
- # Add suffix for 'latest' releases (rc already has suffix in tag)
- if [ "$RELEASE_TYPE" = "latest" ]; then
- CHART_VERSION="${CHART_VERSION}-latest"
- fi
-
- # App version = Docker tag (keeps 'v' prefix to match Docker image tags)
- APP_VERSION="${INPUT_TAG}"
-
- echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
- echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
-
- - uses: ./.github/actions/helm-oci-chart-releaser
- with:
- name: ${{ env.CHART_NAME }}
- repository: ${{ env.REPO_OWNER }}
- tag: ${{ steps.chart_version.outputs.version }}
- app_version: ${{ steps.chart_version.outputs.app_version }}
- path: deploy/charts/${{ env.CHART_NAME }}
- registry: ${{ env.REGISTRY }}
- registry_username: ${{ github.actor }}
- registry_password: ${{ secrets.GITHUB_TOKEN }}
- update_dependencies: true
-
- release:
- name: "New LiteLLM Release"
- needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
-
- runs-on: "ubuntu-latest"
-
- steps:
- - name: Display version
- run: echo "Current version is ${{ github.event.inputs.tag }}"
- - name: "Set Release Tag"
- run: echo "RELEASE_TAG=${{ github.event.inputs.tag }}" >> $GITHUB_ENV
- - name: Display release tag
- run: echo "RELEASE_TAG is $RELEASE_TAG"
- - name: "Create release"
- uses: "actions/github-script@v6"
- with:
- github-token: "${{ secrets.GITHUB_TOKEN }}"
- script: |
- const commitHash = "${{ github.event.inputs.commit_hash}}";
- console.log("Commit Hash:", commitHash); // Add this line for debugging
- try {
- const response = await github.rest.repos.createRelease({
- draft: false,
- generate_release_notes: true,
- target_commitish: commitHash,
- name: process.env.RELEASE_TAG,
- owner: context.repo.owner,
- prerelease: false,
- repo: context.repo.repo,
- tag_name: process.env.RELEASE_TAG,
- });
-
- core.exportVariable('RELEASE_ID', response.data.id);
- core.exportVariable('RELEASE_UPLOAD_URL', response.data.upload_url);
- } catch (error) {
- core.setFailed(error.message);
- }
- - name: Fetch Release Notes
- id: release-notes
- uses: actions/github-script@v6
- with:
- github-token: "${{ secrets.GITHUB_TOKEN }}"
- script: |
- try {
- const response = await github.rest.repos.getRelease({
- owner: context.repo.owner,
- repo: context.repo.repo,
- release_id: process.env.RELEASE_ID,
- });
- const formattedBody = JSON.stringify(response.data.body).slice(1, -1);
- return formattedBody;
- } catch (error) {
- core.setFailed(error.message);
- }
- env:
- RELEASE_ID: ${{ env.RELEASE_ID }}
- - name: Github Releases To Discord
- env:
- WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }}
- REALEASE_TAG: ${{ env.RELEASE_TAG }}
- RELEASE_NOTES: ${{ steps.release-notes.outputs.result }}
- run: |
- curl -H "Content-Type: application/json" -X POST -d '{
- "content": "New LiteLLM release '"${RELEASE_TAG}"'",
- "username": "Release Changelog",
- "avatar_url": "https://cdn.discordapp.com/avatars/487431320314576937/bd64361e4ba6313d561d54e78c9e7171.png",
- "embeds": [
- {
- "title": "Changelog for LiteLLM '"${RELEASE_TAG}"'",
- "description": "'"${RELEASE_NOTES}"'",
- "color": 2105893
- }
- ]
- }' $WEBHOOK_URL
-
diff --git a/.github/workflows/ghcr_helm_deploy.yml b/.github/workflows/ghcr_helm_deploy.yml
deleted file mode 100644
index 21b2eaafe19..00000000000
--- a/.github/workflows/ghcr_helm_deploy.yml
+++ /dev/null
@@ -1,67 +0,0 @@
-# Standalone workflow to publish LiteLLM Helm Chart
-# Note: The main ghcr_deploy.yml workflow also publishes the Helm chart as part of a full release
-name: Build, Publish LiteLLM Helm Chart. New Release
-on:
- workflow_dispatch:
- inputs:
- tag:
- description: "LiteLLM version tag (e.g., v1.81.0)"
- required: true
-
-# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
-env:
- REGISTRY: ghcr.io
- IMAGE_NAME: ${{ github.repository }}
- REPO_OWNER: ${{github.repository_owner}}
-
-# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
-jobs:
- build-and-push-helm-chart:
- runs-on: ubuntu-latest
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
-
- - name: Log in to the Container registry
- uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
- with:
- registry: ${{ env.REGISTRY }}
- username: ${{ github.actor }}
- password: ${{ secrets.GITHUB_TOKEN }}
-
- - name: lowercase github.repository_owner
- run: |
- echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
-
- # Sync Helm chart version with LiteLLM release version (1-1 versioning)
- - name: Calculate chart and app versions
- id: chart_version
- shell: bash
- run: |
- INPUT_TAG="${{ github.event.inputs.tag }}"
-
- # Chart version = LiteLLM version without 'v' prefix
- # v1.81.0 -> 1.81.0
- CHART_VERSION="${INPUT_TAG#v}"
-
- # App version = Docker tag (keeps 'v' prefix)
- APP_VERSION="${INPUT_TAG}"
-
- echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
- echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
-
- - name: Lint helm chart
- run: helm lint deploy/charts/litellm-helm
-
- - uses: ./.github/actions/helm-oci-chart-releaser
- with:
- name: litellm-helm
- repository: ${{ env.REPO_OWNER }}
- tag: ${{ steps.chart_version.outputs.version }}
- app_version: ${{ steps.chart_version.outputs.app_version }}
- path: deploy/charts/litellm-helm
- registry: ${{ env.REGISTRY }}
- registry_username: ${{ github.actor }}
- registry_password: ${{ secrets.GITHUB_TOKEN }}
- update_dependencies: true
-
diff --git a/.github/workflows/helm_unit_test.yml b/.github/workflows/helm_unit_test.yml
index c4b83af70a1..06836b1d1cd 100644
--- a/.github/workflows/helm_unit_test.yml
+++ b/.github/workflows/helm_unit_test.yml
@@ -6,22 +6,36 @@ on:
branches:
- main
+permissions:
+ contents: read
+
jobs:
unit-test:
runs-on: ubuntu-latest
steps:
- name: Checkout
- uses: actions/checkout@v2
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
- name: Set up Helm 3.11.1
- uses: azure/setup-helm@v1
+ uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
with:
- version: '3.11.1'
+ version: "3.11.1"
- name: Install Helm Unit Test Plugin
run: |
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4
+ - name: Verify Helm Unit Test Plugin integrity
+ run: |
+ EXPECTED_SHA="e251ba198448629678ff2168e1a469249d998155"
+ PLUGIN_DIR="$(helm env HELM_PLUGINS)/helm-unittest"
+ ACTUAL_SHA="$(git -C "$PLUGIN_DIR" rev-parse HEAD)"
+ if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then
+ echo "::error::Helm unittest plugin checksum mismatch! Expected $EXPECTED_SHA but got $ACTUAL_SHA"
+ exit 1
+ fi
+ echo "Helm unittest plugin integrity verified: $ACTUAL_SHA"
- name: Run unit tests
- run:
- helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
\ No newline at end of file
+ run: helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
diff --git a/.github/workflows/interpret_load_test.py b/.github/workflows/interpret_load_test.py
deleted file mode 100644
index 0b5df738626..00000000000
--- a/.github/workflows/interpret_load_test.py
+++ /dev/null
@@ -1,139 +0,0 @@
-import csv
-import os
-from github import Github
-
-
-def interpret_results(csv_file):
- with open(csv_file, newline="") as csvfile:
- csvreader = csv.DictReader(csvfile)
- rows = list(csvreader)
- """
- in this csv reader
- - Create 1 new column "Status"
- - if a row has a median response time < 300 and an average response time < 300, Status = "Passed ✅"
- - if a row has a median response time >= 300 or an average response time >= 300, Status = "Failed ❌"
- - Order the table in this order Name, Status, Median Response Time, Average Response Time, Requests/s,Failures/s, Min Response Time, Max Response Time, all other columns
- """
-
- # Add a new column "Status"
- for row in rows:
- median_response_time = float(
- row["Median Response Time"].strip().rstrip("ms")
- )
- average_response_time = float(
- row["Average Response Time"].strip().rstrip("s")
- )
-
- request_count = int(row["Request Count"])
- failure_count = int(row["Failure Count"])
-
- failure_percent = round((failure_count / request_count) * 100, 2)
-
- # Determine status based on conditions
- if (
- median_response_time < 300
- and average_response_time < 300
- and failure_percent < 5
- ):
- row["Status"] = "Passed ✅"
- else:
- row["Status"] = "Failed ❌"
-
- # Construct Markdown table header
- markdown_table = "| Name | Status | Median Response Time (ms) | Average Response Time (ms) | Requests/s | Failures/s | Request Count | Failure Count | Min Response Time (ms) | Max Response Time (ms) |"
- markdown_table += (
- "\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"
- )
-
- # Construct Markdown table rows
- for row in rows:
- markdown_table += f"\n| {row['Name']} | {row['Status']} | {row['Median Response Time']} | {row['Average Response Time']} | {row['Requests/s']} | {row['Failures/s']} | {row['Request Count']} | {row['Failure Count']} | {row['Min Response Time']} | {row['Max Response Time']} |"
- print("markdown table: ", markdown_table)
- return markdown_table
-
-
-def _get_docker_run_command_stable_release(release_version):
- return f"""
-\n\n
-## Docker Run LiteLLM Proxy
-
-```
-docker run \\
--e STORE_MODEL_IN_DB=True \\
--p 4000:4000 \\
-ghcr.io/berriai/litellm:litellm_stable_release_branch-{release_version}
-```
- """
-
-
-def _get_docker_run_command(release_version):
- return f"""
-\n\n
-## Docker Run LiteLLM Proxy
-
-```
-docker run \\
--e STORE_MODEL_IN_DB=True \\
--p 4000:4000 \\
-ghcr.io/berriai/litellm:main-{release_version}
-```
- """
-
-
-def get_docker_run_command(release_version):
- if "stable" in release_version:
- return _get_docker_run_command_stable_release(release_version)
- else:
- return _get_docker_run_command(release_version)
-
-
-if __name__ == "__main__":
- return
- csv_file = "load_test_stats.csv" # Change this to the path of your CSV file
- markdown_table = interpret_results(csv_file)
-
- # Update release body with interpreted results
- github_token = os.getenv("GITHUB_TOKEN")
- g = Github(github_token)
- repo = g.get_repo(
- "BerriAI/litellm"
- ) # Replace with your repository's username and name
- latest_release = repo.get_latest_release()
- print("got latest release: ", latest_release)
- print(latest_release.title)
- print(latest_release.tag_name)
-
- release_version = latest_release.title
-
- print("latest release body: ", latest_release.body)
- print("markdown table: ", markdown_table)
-
- # check if "Load Test LiteLLM Proxy Results" exists
- existing_release_body = latest_release.body
- if "Load Test LiteLLM Proxy Results" in latest_release.body:
- # find the "Load Test LiteLLM Proxy Results" section and delete it
- start_index = latest_release.body.find("Load Test LiteLLM Proxy Results")
- existing_release_body = latest_release.body[:start_index]
-
- docker_run_command = get_docker_run_command(release_version)
- print("docker run command: ", docker_run_command)
-
- new_release_body = (
- existing_release_body
- + docker_run_command
- + "\n\n"
- + "### Don't want to maintain your internal proxy? get in touch 🎉"
- + "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat"
- + "\n\n"
- + "## Load Test LiteLLM Proxy Results"
- + "\n\n"
- + markdown_table
- )
- print("new release body: ", new_release_body)
- try:
- latest_release.update_release(
- name=latest_release.tag_name,
- message=new_release_body,
- )
- except Exception as e:
- print(e)
diff --git a/.github/workflows/issue-keyword-labeler.yml b/.github/workflows/issue-keyword-labeler.yml
index 936f90f747f..7e2693209b6 100644
--- a/.github/workflows/issue-keyword-labeler.yml
+++ b/.github/workflows/issue-keyword-labeler.yml
@@ -2,8 +2,8 @@ name: Issue Keyword Labeler
on:
issues:
- types:
- - opened
+ types:
+ - opened
jobs:
scan-and-label:
@@ -13,7 +13,9 @@ jobs:
contents: read
steps:
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
- name: Scan for provider keywords
id: scan
@@ -24,7 +26,7 @@ jobs:
- name: Ensure label exists
if: steps.scan.outputs.found == 'true'
- uses: actions/github-script@v7
+ uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
@@ -51,7 +53,7 @@ jobs:
- name: Add label to the issue
if: steps.scan.outputs.found == 'true'
- uses: actions/github-script@v7
+ uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
@@ -61,4 +63,3 @@ jobs:
issue_number: context.issue.number,
labels: ['llm translation']
});
-
diff --git a/.github/workflows/label-component.yml b/.github/workflows/label-component.yml
index fd079fce6c1..e0c2fa94d8c 100644
--- a/.github/workflows/label-component.yml
+++ b/.github/workflows/label-component.yml
@@ -12,7 +12,7 @@ jobs:
issues: write
steps:
- name: Add component labels
- uses: actions/github-script@v7
+ uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
diff --git a/.github/workflows/llm-translation-testing.yml b/.github/workflows/llm-translation-testing.yml
index 7fda37a66dc..922013c4b54 100644
--- a/.github/workflows/llm-translation-testing.yml
+++ b/.github/workflows/llm-translation-testing.yml
@@ -4,38 +4,41 @@ on:
workflow_dispatch:
inputs:
release_candidate_tag:
- description: 'Release candidate tag/version'
+ description: "Release candidate tag/version"
required: true
type: string
push:
tags:
- - 'v*-rc*' # Triggers on release candidate tags like v1.0.0-rc1
-
+ - "v*-rc*" # Triggers on release candidate tags like v1.0.0-rc1
+
+permissions:
+ contents: read
+
jobs:
run-llm-translation-tests:
runs-on: ubuntu-latest
timeout-minutes: 90
-
+
steps:
- name: Checkout code
- uses: actions/checkout@v4
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
+ persist-credentials: false
ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}
-
+
- name: Set up Python
- uses: actions/setup-python@v5
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
- python-version: '3.11'
-
+ python-version: "3.11"
+
- name: Install Poetry
- uses: snok/install-poetry@v1
- with:
- version: latest
- virtualenvs-create: true
- virtualenvs-in-project: true
-
- - name: Cache Poetry dependencies
- uses: actions/cache@v3
+ run: |
+ pip install 'poetry==2.3.2'
+ poetry config virtualenvs.create true
+ poetry config virtualenvs.in-project true
+
+ - name: Restore Poetry dependencies cache
+ uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0
with:
path: |
~/.cache/pypoetry
@@ -43,15 +46,15 @@ jobs:
key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
restore-keys: |
${{ runner.os }}-poetry-
-
+
- name: Install dependencies
run: |
poetry install --with dev
- poetry run pip install pytest-xdist pytest-timeout
-
+ poetry run pip install 'pytest-xdist==3.8.0' 'pytest-timeout==2.4.0'
+
- name: Create test results directory
run: mkdir -p test-results
-
+
- name: Run LLM Translation Tests
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -61,13 +64,14 @@ jobs:
AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
- # Add other API keys as needed
+ RC_TAG: ${{ github.event.inputs.release_candidate_tag || github.ref_name }}
+ COMMIT_SHA: ${{ github.sha }}
run: |
python .github/workflows/run_llm_translation_tests.py \
- --tag "${{ github.event.inputs.release_candidate_tag || github.ref_name }}" \
- --commit "${{ github.sha }}" \
+ --tag "$RC_TAG" \
+ --commit "$COMMIT_SHA" \
|| true # Continue even if tests fail
-
+
- name: Display test summary
if: always()
run: |
@@ -79,9 +83,9 @@ jobs:
else
echo "Warning: Test report was not generated"
fi
-
+
- name: Upload test artifacts
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml
deleted file mode 100644
index cdaffa328c9..00000000000
--- a/.github/workflows/load_test.yml
+++ /dev/null
@@ -1,59 +0,0 @@
-name: Test Locust Load Test
-
-on:
- workflow_run:
- workflows: ["Build, Publish LiteLLM Docker Image. New Release"]
- types:
- - completed
- workflow_dispatch:
-
-jobs:
- build:
- runs-on: ubuntu-latest
- steps:
- - name: Checkout
- uses: actions/checkout@v1
- - name: Setup Python
- uses: actions/setup-python@v2
- with:
- python-version: '3.x'
-
- - name: Install dependencies
- run: |
- python -m pip install --upgrade pip
- pip install PyGithub
- - name: re-deploy proxy
- run: |
- echo "Current working directory: $PWD"
- ls
- python ".github/workflows/redeploy_proxy.py"
- env:
- LOAD_TEST_REDEPLOY_URL1: ${{ secrets.LOAD_TEST_REDEPLOY_URL1 }}
- LOAD_TEST_REDEPLOY_URL2: ${{ secrets.LOAD_TEST_REDEPLOY_URL2 }}
- working-directory: ${{ github.workspace }}
- - name: Run Load Test
- id: locust_run
- uses: BerriAI/locust-github-action@master
- with:
- LOCUSTFILE: ".github/workflows/locustfile.py"
- URL: "https://post-release-load-test-proxy.onrender.com/"
- USERS: "20"
- RATE: "20"
- RUNTIME: "300s"
- - name: Process Load Test Stats
- run: |
- echo "Current working directory: $PWD"
- ls
- python ".github/workflows/interpret_load_test.py"
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- working-directory: ${{ github.workspace }}
- - name: Upload CSV as Asset to Latest Release
- uses: xresloader/upload-to-github-release@v1
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- with:
- file: "load_test_stats.csv;load_test.html"
- update_latest_release: true
- tag_name: "load-test"
- overwrite: true
\ No newline at end of file
diff --git a/.github/workflows/locustfile.py b/.github/workflows/locustfile.py
deleted file mode 100644
index 36dbeee9c48..00000000000
--- a/.github/workflows/locustfile.py
+++ /dev/null
@@ -1,28 +0,0 @@
-from locust import HttpUser, task, between
-
-
-class MyUser(HttpUser):
- wait_time = between(1, 5)
-
- @task
- def chat_completion(self):
- headers = {
- "Content-Type": "application/json",
- "Authorization": "Bearer sk-8N1tLOOyH8TIxwOLahhIVg",
- # Include any additional headers you may need for authentication, etc.
- }
-
- # Customize the payload with "model" and "messages" keys
- payload = {
- "model": "fake-openai-endpoint",
- "messages": [
- {"role": "system", "content": "You are a chat bot."},
- {"role": "user", "content": "Hello, how are you?"},
- ],
- # Add more data as necessary
- }
-
- # Make a POST request to the "chat/completions" endpoint
- response = self.client.post("chat/completions", json=payload, headers=headers)
-
- # Print or log the response if needed
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
deleted file mode 100644
index 23e4a06da9e..00000000000
--- a/.github/workflows/main.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-name: Publish Dev Release to PyPI
-
-on:
- workflow_dispatch:
-
-jobs:
- publish-dev-release:
- runs-on: ubuntu-latest
-
- steps:
- - name: Checkout code
- uses: actions/checkout@v2
-
- - name: Set up Python
- uses: actions/setup-python@v2
- with:
- python-version: 3.8 # Adjust the Python version as needed
-
- - name: Install dependencies
- run: pip install toml twine
-
- - name: Read version from pyproject.toml
- id: read-version
- run: |
- version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])')
- printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV
-
- - name: Check if version exists on PyPI
- id: check-version
- run: |
- set -e
- if twine check --repository-url https://pypi.org/simple/ "litellm==$LITELLM_VERSION" >/dev/null 2>&1; then
- echo "Version $LITELLM_VERSION already exists on PyPI. Skipping publish."
-
diff --git a/.github/workflows/publish-migrations.yml b/.github/workflows/publish-migrations.yml
deleted file mode 100644
index a5187cb2f55..00000000000
--- a/.github/workflows/publish-migrations.yml
+++ /dev/null
@@ -1,207 +0,0 @@
-name: Publish Prisma Migrations
-
-permissions:
- contents: write
- pull-requests: write
-
-on:
- push:
- paths:
- - 'schema.prisma' # Check root schema.prisma
- branches:
- - main
-
-jobs:
- publish-migrations:
- if: github.repository == 'BerriAI/litellm'
- runs-on: ubuntu-latest
- services:
- postgres:
- image: postgres:14
- env:
- POSTGRES_DB: temp_db
- POSTGRES_USER: postgres
- POSTGRES_PASSWORD: postgres
- ports:
- - 5432:5432
- options: >-
- --health-cmd pg_isready
- --health-interval 10s
- --health-timeout 5s
- --health-retries 5
-
- # Add shadow database service
- postgres_shadow:
- image: postgres:14
- env:
- POSTGRES_DB: shadow_db
- POSTGRES_USER: postgres
- POSTGRES_PASSWORD: postgres
- ports:
- - 5433:5432
- options: >-
- --health-cmd pg_isready
- --health-interval 10s
- --health-timeout 5s
- --health-retries 5
-
- steps:
- - uses: actions/checkout@v3
-
- - name: Set up Python
- uses: actions/setup-python@v4
- with:
- python-version: '3.x'
-
- - name: Install Dependencies
- run: |
- pip install prisma
- pip install python-dotenv
-
- - name: Generate Initial Migration if None Exists
- env:
- DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
- DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
- SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
- run: |
- mkdir -p deploy/migrations
- echo 'provider = "postgresql"' > deploy/migrations/migration_lock.toml
-
- if [ -z "$(ls -A deploy/migrations/2* 2>/dev/null)" ]; then
- echo "No existing migrations found, creating baseline..."
- VERSION=$(date +%Y%m%d%H%M%S)
- mkdir -p deploy/migrations/${VERSION}_initial
-
- echo "Generating initial migration..."
- # Save raw output for debugging
- prisma migrate diff \
- --from-empty \
- --to-schema-datamodel schema.prisma \
- --shadow-database-url "${SHADOW_DATABASE_URL}" \
- --script > deploy/migrations/${VERSION}_initial/raw_migration.sql
-
- echo "Raw migration file content:"
- cat deploy/migrations/${VERSION}_initial/raw_migration.sql
-
- echo "Cleaning migration file..."
- # Clean the file
- sed '/^Installing/d' deploy/migrations/${VERSION}_initial/raw_migration.sql > deploy/migrations/${VERSION}_initial/migration.sql
-
- # Verify the migration file
- if [ ! -s deploy/migrations/${VERSION}_initial/migration.sql ]; then
- echo "ERROR: Migration file is empty after cleaning"
- echo "Original content was:"
- cat deploy/migrations/${VERSION}_initial/raw_migration.sql
- exit 1
- fi
-
- echo "Final migration file content:"
- cat deploy/migrations/${VERSION}_initial/migration.sql
-
- # Verify it starts with SQL
- if ! head -n 1 deploy/migrations/${VERSION}_initial/migration.sql | grep -q "^--\|^CREATE\|^ALTER"; then
- echo "ERROR: Migration file does not start with SQL command or comment"
- echo "First line is:"
- head -n 1 deploy/migrations/${VERSION}_initial/migration.sql
- echo "Full content is:"
- cat deploy/migrations/${VERSION}_initial/migration.sql
- exit 1
- fi
-
- echo "Initial migration generated at $(date -u)" > deploy/migrations/${VERSION}_initial/README.md
- fi
-
- - name: Compare and Generate Migration
- if: success()
- env:
- DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
- DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
- SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
- run: |
- # Create temporary migration workspace
- mkdir -p temp_migrations
-
- # Copy existing migrations (will not fail if directory is empty)
- cp -r deploy/migrations/* temp_migrations/ 2>/dev/null || true
-
- VERSION=$(date +%Y%m%d%H%M%S)
-
- # Generate diff against existing migrations or empty state
- prisma migrate diff \
- --from-migrations temp_migrations \
- --to-schema-datamodel schema.prisma \
- --shadow-database-url "${SHADOW_DATABASE_URL}" \
- --script > temp_migrations/migration_${VERSION}.sql
-
- # Check if there are actual changes
- if [ -s temp_migrations/migration_${VERSION}.sql ]; then
- echo "Changes detected, creating new migration"
- mkdir -p deploy/migrations/${VERSION}_schema_update
- mv temp_migrations/migration_${VERSION}.sql deploy/migrations/${VERSION}_schema_update/migration.sql
- echo "Migration generated at $(date -u)" > deploy/migrations/${VERSION}_schema_update/README.md
- else
- echo "No schema changes detected"
- exit 0
- fi
-
- - name: Verify Migration
- if: success()
- env:
- DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
- DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
- SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
- run: |
- # Create test database
- psql "${SHADOW_DATABASE_URL}" -c 'CREATE DATABASE migration_test;'
-
- # Apply all migrations in order to verify
- for migration in deploy/migrations/*/migration.sql; do
- echo "Applying migration: $migration"
- psql "${SHADOW_DATABASE_URL}" -f $migration
- done
-
- # Add this step before create-pull-request to debug permissions
- - name: Check Token Permissions
- run: |
- echo "Checking token permissions..."
- curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
- -H "Accept: application/vnd.github.v3+json" \
- https://api.github.com/repos/BerriAI/litellm/collaborators
-
- echo "\nChecking if token can create PRs..."
- curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
- -H "Accept: application/vnd.github.v3+json" \
- https://api.github.com/repos/BerriAI/litellm
-
- # Add this debug step before git push
- - name: Debug Changed Files
- run: |
- echo "Files staged for commit:"
- git diff --name-status --staged
-
- echo "\nAll changed files:"
- git status
-
- - name: Create Pull Request
- if: success()
- uses: peter-evans/create-pull-request@v5
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- commit-message: "chore: update prisma migrations"
- title: "Update Prisma Migrations"
- body: |
- Auto-generated migration based on schema.prisma changes.
-
- Generated files:
- - deploy/migrations/${VERSION}_schema_update/migration.sql
- - deploy/migrations/${VERSION}_schema_update/README.md
- branch: feat/prisma-migration-${{ env.VERSION }}
- base: main
- delete-branch: true
-
- - name: Generate and Save Migrations
- run: |
- # Only add migration files
- git add deploy/migrations/
- git status # Debug what's being committed
- git commit -m "chore: update prisma migrations"
diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml
new file mode 100644
index 00000000000..8f675bb3075
--- /dev/null
+++ b/.github/workflows/publish_to_pypi.yml
@@ -0,0 +1,136 @@
+name: Publish to PyPI
+
+on:
+ workflow_dispatch:
+
+jobs:
+ preflight-checks:
+ name: Preflight Checks
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ permissions:
+ contents: read
+ # No environment — read-only checks, no approval needed
+ outputs:
+ needs_publish: ${{ steps.check-litellm.outputs.needs_publish }}
+ version: ${{ steps.check-litellm.outputs.version }}
+
+ steps:
+ - name: Checkout repo
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+
+ - name: Check litellm version on PyPI
+ id: check-litellm
+ run: |
+ VERSION=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/')
+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+ echo "Checking if litellm $VERSION exists on PyPI..."
+
+ HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm/$VERSION/json")
+ if [ "$HTTP_STATUS" = "200" ]; then
+ echo "litellm $VERSION already exists on PyPI. Skipping publish."
+ echo "needs_publish=false" >> "$GITHUB_OUTPUT"
+ else
+ echo "litellm $VERSION not found on PyPI. Publish needed."
+ echo "needs_publish=true" >> "$GITHUB_OUTPUT"
+ fi
+
+ - name: Sanity check proxy-extras version
+ run: |
+ # Read pinned version from requirements.txt
+ REQ_VERSION=$(grep -oP 'litellm-proxy-extras==\K[0-9.]+' requirements.txt)
+ if [ -z "$REQ_VERSION" ]; then
+ echo "::error::Could not find litellm-proxy-extras version in requirements.txt"
+ exit 1
+ fi
+ echo "requirements.txt pins litellm-proxy-extras==$REQ_VERSION"
+
+ # Read pinned version from pyproject.toml dependency
+ PYPROJECT_VERSION=$(python3 -c "
+ import re
+ with open('pyproject.toml') as f:
+ content = f.read()
+ match = re.search(r'litellm-proxy-extras\s*=\s*\{version\s*=\s*\"([^\"]+)\"', content)
+ if match:
+ print(match.group(1).lstrip('^~>='))
+ else:
+ import sys
+ print('::error::Could not find litellm-proxy-extras dependency in pyproject.toml', file=sys.stderr)
+ sys.exit(1)
+ ")
+ echo "pyproject.toml pins litellm-proxy-extras version: $PYPROJECT_VERSION"
+
+ # Check that both pinned versions match
+ if [ "$REQ_VERSION" != "$PYPROJECT_VERSION" ]; then
+ echo "::error::Version mismatch: requirements.txt has $REQ_VERSION but pyproject.toml has $PYPROJECT_VERSION"
+ exit 1
+ fi
+
+ # Check that the pinned version exists on PyPI
+ echo "Checking if litellm-proxy-extras $REQ_VERSION exists on PyPI..."
+ HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$REQ_VERSION/json")
+ if [ "$HTTP_STATUS" != "200" ]; then
+ echo "::error::litellm-proxy-extras $REQ_VERSION is not published on PyPI yet. Publish it before releasing litellm."
+ exit 1
+ fi
+ echo "litellm-proxy-extras $REQ_VERSION exists on PyPI. Sanity check passed."
+
+ publish-litellm:
+ name: Publish litellm to PyPI
+ needs: preflight-checks
+ if: needs.preflight-checks.outputs.needs_publish == 'true'
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ permissions:
+ id-token: write
+ contents: read
+ environment: pypi-publish
+
+ steps:
+ - name: Checkout repo
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+
+ - name: Copy model prices backup
+ run: cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
+
+ - name: Install build tools
+ run: python -m pip install --upgrade pip build==1.4.2
+
+ - name: Build package
+ run: |
+ rm -rf build dist
+ python -m build
+
+ - name: Verify build artifacts
+ env:
+ EXPECTED_VERSION: ${{ needs.preflight-checks.outputs.version }}
+ run: |
+ echo "Contents of dist/:"
+ ls -la dist/
+ # Ensure we have both sdist and wheel
+ ls dist/*.tar.gz
+ ls dist/*.whl
+ # Verify built version matches expected
+ ls dist/ | grep -q "litellm-${EXPECTED_VERSION}" || {
+ echo "::error::Built artifacts do not match expected version $EXPECTED_VERSION"
+ ls dist/
+ exit 1
+ }
+
+ - name: Validate package metadata
+ run: |
+ pip install twine==6.2.0
+ twine check dist/*
+
+ - name: Publish to PyPI
+ uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
diff --git a/.github/workflows/read_pyproject_version.yml b/.github/workflows/read_pyproject_version.yml
index 8f6310f935b..04b4a38ce19 100644
--- a/.github/workflows/read_pyproject_version.yml
+++ b/.github/workflows/read_pyproject_version.yml
@@ -3,7 +3,10 @@ name: Read Version from pyproject.toml
on:
push:
branches:
- - main # Change this to the default branch of your repository
+ - main # Change this to the default branch of your repository
+
+permissions:
+ contents: read
jobs:
read-version:
@@ -11,20 +14,14 @@ jobs:
steps:
- name: Checkout code
- uses: actions/checkout@v2
-
- - name: Set up Python
- uses: actions/setup-python@v2
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
- python-version: 3.8 # Adjust the Python version as needed
-
- - name: Install dependencies
- run: pip install toml
+ persist-credentials: false
- name: Read version from pyproject.toml
id: read-version
run: |
- version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])')
+ version=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/')
printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV
- name: Display version
diff --git a/.github/workflows/redeploy_proxy.py b/.github/workflows/redeploy_proxy.py
deleted file mode 100644
index ed46bef73a2..00000000000
--- a/.github/workflows/redeploy_proxy.py
+++ /dev/null
@@ -1,20 +0,0 @@
-"""
-
-redeploy_proxy.py
-"""
-
-import os
-import requests
-import time
-
-# send a get request to this endpoint
-deploy_hook1 = os.getenv("LOAD_TEST_REDEPLOY_URL1")
-response = requests.get(deploy_hook1, timeout=20)
-
-
-deploy_hook2 = os.getenv("LOAD_TEST_REDEPLOY_URL2")
-response = requests.get(deploy_hook2, timeout=20)
-
-print("SENT GET REQUESTS to re-deploy proxy")
-print("sleeeping.... for 60s")
-time.sleep(60)
diff --git a/.github/workflows/reset_stable.yml b/.github/workflows/reset_stable.yml
deleted file mode 100644
index f6fed672d47..00000000000
--- a/.github/workflows/reset_stable.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-name: Reset litellm_stable branch
-
-on:
- release:
- types: [published, created]
-jobs:
- update-stable-branch:
- if: ${{ startsWith(github.event.release.tag_name, 'v') && !endsWith(github.event.release.tag_name, '-stable') }}
- runs-on: ubuntu-latest
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@v3
-
- - name: Reset litellm_stable_release_branch branch to the release commit
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- # Configure Git user
- git config user.name "github-actions[bot]"
- git config user.email "github-actions[bot]@users.noreply.github.com"
-
- # Fetch all branches and tags
- git fetch --all
-
- # Check if the litellm_stable_release_branch branch exists
- if git show-ref --verify --quiet refs/remotes/origin/litellm_stable_release_branch; then
- echo "litellm_stable_release_branch branch exists."
- git checkout litellm_stable_release_branch
- else
- echo "litellm_stable_release_branch branch does not exist. Creating it."
- git checkout -b litellm_stable_release_branch
- fi
-
- # Reset litellm_stable_release_branch branch to the release commit
- git reset --hard $GITHUB_SHA
-
- # Push the updated litellm_stable_release_branch branch
- git push origin litellm_stable_release_branch --force
diff --git a/.github/workflows/run_llm_translation_tests.py b/.github/workflows/run_llm_translation_tests.py
old mode 100755
new mode 100644
index 70e3c618275..5b3a4817ecb
--- a/.github/workflows/run_llm_translation_tests.py
+++ b/.github/workflows/run_llm_translation_tests.py
@@ -12,76 +12,68 @@ import subprocess
import xml.etree.ElementTree as ET
from collections import defaultdict
from datetime import datetime
-
+from pathlib import Path
+import json
+from typing import Dict, List, Tuple, Optional
# ANSI color codes for terminal output
class Colors:
- GREEN = "\033[92m"
- RED = "\033[91m"
- YELLOW = "\033[93m"
- BLUE = "\033[94m"
- PURPLE = "\033[95m"
- CYAN = "\033[96m"
- RESET = "\033[0m"
- BOLD = "\033[1m"
-
+ GREEN = '\033[92m'
+ RED = '\033[91m'
+ YELLOW = '\033[93m'
+ BLUE = '\033[94m'
+ PURPLE = '\033[95m'
+ CYAN = '\033[96m'
+ RESET = '\033[0m'
+ BOLD = '\033[1m'
def print_colored(message: str, color: str = Colors.RESET):
"""Print colored message to terminal"""
print(f"{color}{message}{Colors.RESET}")
-
def get_provider_from_test_file(test_file: str) -> str:
"""Map test file names to provider names"""
provider_mapping = {
- "test_anthropic": "Anthropic",
- "test_azure": "Azure",
- "test_bedrock": "AWS Bedrock",
- "test_openai": "OpenAI",
- "test_vertex": "Google Vertex AI",
- "test_gemini": "Google Vertex AI",
- "test_cohere": "Cohere",
- "test_databricks": "Databricks",
- "test_groq": "Groq",
- "test_together": "Together AI",
- "test_mistral": "Mistral",
- "test_deepseek": "DeepSeek",
- "test_replicate": "Replicate",
- "test_huggingface": "HuggingFace",
- "test_fireworks": "Fireworks AI",
- "test_perplexity": "Perplexity",
- "test_cloudflare": "Cloudflare",
- "test_voyage": "Voyage AI",
- "test_xai": "xAI",
- "test_nvidia": "NVIDIA",
- "test_watsonx": "IBM watsonx",
- "test_azure_ai": "Azure AI",
- "test_snowflake": "Snowflake",
- "test_infinity": "Infinity",
- "test_jina": "Jina AI",
- "test_deepgram": "Deepgram",
- "test_clarifai": "Clarifai",
- "test_triton": "Triton",
+ 'test_anthropic': 'Anthropic',
+ 'test_azure': 'Azure',
+ 'test_bedrock': 'AWS Bedrock',
+ 'test_openai': 'OpenAI',
+ 'test_vertex': 'Google Vertex AI',
+ 'test_gemini': 'Google Vertex AI',
+ 'test_cohere': 'Cohere',
+ 'test_databricks': 'Databricks',
+ 'test_groq': 'Groq',
+ 'test_together': 'Together AI',
+ 'test_mistral': 'Mistral',
+ 'test_deepseek': 'DeepSeek',
+ 'test_replicate': 'Replicate',
+ 'test_huggingface': 'HuggingFace',
+ 'test_fireworks': 'Fireworks AI',
+ 'test_perplexity': 'Perplexity',
+ 'test_cloudflare': 'Cloudflare',
+ 'test_voyage': 'Voyage AI',
+ 'test_xai': 'xAI',
+ 'test_nvidia': 'NVIDIA',
+ 'test_watsonx': 'IBM watsonx',
+ 'test_azure_ai': 'Azure AI',
+ 'test_snowflake': 'Snowflake',
+ 'test_infinity': 'Infinity',
+ 'test_jina': 'Jina AI',
+ 'test_deepgram': 'Deepgram',
+ 'test_clarifai': 'Clarifai',
+ 'test_triton': 'Triton',
}
-
+
for key, provider in provider_mapping.items():
if key in test_file:
return provider
-
+
# For cross-provider test files
- if any(
- name in test_file
- for name in [
- "test_optional_params",
- "test_prompt_factory",
- "test_router",
- "test_text_completion",
- ]
- ):
- return f"Cross-Provider Tests ({test_file})"
-
- return "Other Tests"
-
+ if any(name in test_file for name in ['test_optional_params', 'test_prompt_factory',
+ 'test_router', 'test_text_completion']):
+ return f'Cross-Provider Tests ({test_file})'
+
+ return 'Other Tests'
def format_duration(seconds: float) -> str:
"""Format duration in human-readable format"""
@@ -97,354 +89,290 @@ def format_duration(seconds: float) -> str:
return f"{hours}h {minutes}m"
-def generate_markdown_report(
- junit_xml_path: str, output_path: str, tag: str = None, commit: str = None
-):
+def generate_markdown_report(junit_xml_path: str, output_path: str, tag: str = None, commit: str = None):
"""Generate a beautiful markdown report from JUnit XML"""
try:
tree = ET.parse(junit_xml_path)
root = tree.getroot()
-
+
# Handle both testsuite and testsuites root
- if root.tag == "testsuites":
- suites = root.findall("testsuite")
+ if root.tag == 'testsuites':
+ suites = root.findall('testsuite')
else:
suites = [root]
-
+
# Overall statistics
total_tests = 0
total_failures = 0
total_errors = 0
total_skipped = 0
total_time = 0.0
-
+
# Provider breakdown
- provider_stats = defaultdict(
- lambda: {"passed": 0, "failed": 0, "skipped": 0, "errors": 0, "time": 0.0}
- )
+ provider_stats = defaultdict(lambda: {'passed': 0, 'failed': 0, 'skipped': 0, 'errors': 0, 'time': 0.0})
provider_tests = defaultdict(list)
-
+
for suite in suites:
- total_tests += int(suite.get("tests", 0))
- total_failures += int(suite.get("failures", 0))
- total_errors += int(suite.get("errors", 0))
- total_skipped += int(suite.get("skipped", 0))
- total_time += float(suite.get("time", 0))
-
- for testcase in suite.findall("testcase"):
- classname = testcase.get("classname", "")
- test_name = testcase.get("name", "")
- test_time = float(testcase.get("time", 0))
-
+ total_tests += int(suite.get('tests', 0))
+ total_failures += int(suite.get('failures', 0))
+ total_errors += int(suite.get('errors', 0))
+ total_skipped += int(suite.get('skipped', 0))
+ total_time += float(suite.get('time', 0))
+
+ for testcase in suite.findall('testcase'):
+ classname = testcase.get('classname', '')
+ test_name = testcase.get('name', '')
+ test_time = float(testcase.get('time', 0))
+
# Extract test file name from classname
- if "." in classname:
- parts = classname.split(".")
- test_file = parts[-2] if len(parts) > 1 else "unknown"
+ if '.' in classname:
+ parts = classname.split('.')
+ test_file = parts[-2] if len(parts) > 1 else 'unknown'
else:
- test_file = "unknown"
-
+ test_file = 'unknown'
+
provider = get_provider_from_test_file(test_file)
- provider_stats[provider]["time"] += test_time
-
+ provider_stats[provider]['time'] += test_time
+
# Check test status
- if testcase.find("failure") is not None:
- provider_stats[provider]["failed"] += 1
- failure = testcase.find("failure")
- failure_msg = (
- failure.get("message", "") if failure is not None else ""
- )
- provider_tests[provider].append(
- {
- "name": test_name,
- "status": "FAILED",
- "time": test_time,
- "message": failure_msg,
- }
- )
- elif testcase.find("error") is not None:
- provider_stats[provider]["errors"] += 1
- error = testcase.find("error")
- error_msg = error.get("message", "") if error is not None else ""
- provider_tests[provider].append(
- {
- "name": test_name,
- "status": "ERROR",
- "time": test_time,
- "message": error_msg,
- }
- )
- elif testcase.find("skipped") is not None:
- provider_stats[provider]["skipped"] += 1
- skip = testcase.find("skipped")
- skip_msg = skip.get("message", "") if skip is not None else ""
- provider_tests[provider].append(
- {
- "name": test_name,
- "status": "SKIPPED",
- "time": test_time,
- "message": skip_msg,
- }
- )
+ if testcase.find('failure') is not None:
+ provider_stats[provider]['failed'] += 1
+ failure = testcase.find('failure')
+ failure_msg = failure.get('message', '') if failure is not None else ''
+ provider_tests[provider].append({
+ 'name': test_name,
+ 'status': 'FAILED',
+ 'time': test_time,
+ 'message': failure_msg
+ })
+ elif testcase.find('error') is not None:
+ provider_stats[provider]['errors'] += 1
+ error = testcase.find('error')
+ error_msg = error.get('message', '') if error is not None else ''
+ provider_tests[provider].append({
+ 'name': test_name,
+ 'status': 'ERROR',
+ 'time': test_time,
+ 'message': error_msg
+ })
+ elif testcase.find('skipped') is not None:
+ provider_stats[provider]['skipped'] += 1
+ skip = testcase.find('skipped')
+ skip_msg = skip.get('message', '') if skip is not None else ''
+ provider_tests[provider].append({
+ 'name': test_name,
+ 'status': 'SKIPPED',
+ 'time': test_time,
+ 'message': skip_msg
+ })
else:
- provider_stats[provider]["passed"] += 1
- provider_tests[provider].append(
- {
- "name": test_name,
- "status": "PASSED",
- "time": test_time,
- "message": "",
- }
- )
-
+ provider_stats[provider]['passed'] += 1
+ provider_tests[provider].append({
+ 'name': test_name,
+ 'status': 'PASSED',
+ 'time': test_time,
+ 'message': ''
+ })
+
passed = total_tests - total_failures - total_errors - total_skipped
-
+
# Generate the markdown report
- with open(output_path, "w") as f:
+ with open(output_path, 'w') as f:
# Header
f.write("# LLM Translation Test Results\n\n")
-
+
# Metadata table
f.write("## Test Run Information\n\n")
f.write("| Field | Value |\n")
f.write("|-------|-------|\n")
f.write(f"| **Tag** | `{tag or 'N/A'}` |\n")
- f.write(
- f"| **Date** | {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')} |\n"
- )
+ f.write(f"| **Date** | {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')} |\n")
f.write(f"| **Commit** | `{commit or 'N/A'}` |\n")
f.write(f"| **Duration** | {format_duration(total_time)} |\n")
f.write("\n")
-
+
# Overall statistics with visual elements
f.write("## Overall Statistics\n\n")
-
+
# Summary box
f.write("```\n")
f.write(f"Total Tests: {total_tests}\n")
- f.write(
- f"├── Passed: {passed:>4} ({(passed/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n"
- )
- f.write(
- f"├── Failed: {total_failures:>4} ({(total_failures/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n"
- )
- f.write(
- f"├── Errors: {total_errors:>4} ({(total_errors/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n"
- )
- f.write(
- f"└── Skipped: {total_skipped:>4} ({(total_skipped/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n"
- )
+ f.write(f"├── Passed: {passed:>4} ({(passed/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
+ f.write(f"├── Failed: {total_failures:>4} ({(total_failures/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
+ f.write(f"├── Errors: {total_errors:>4} ({(total_errors/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
+ f.write(f"└── Skipped: {total_skipped:>4} ({(total_skipped/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
f.write("```\n\n")
-
+
+
# Provider summary table
f.write("## Results by Provider\n\n")
- f.write(
- "| Provider | Total | Pass | Fail | Error | Skip | Pass Rate | Duration |\n"
- )
- f.write(
- "|----------|-------|------|------|-------|------|-----------|----------|"
- )
-
+ f.write("| Provider | Total | Pass | Fail | Error | Skip | Pass Rate | Duration |\n")
+ f.write("|----------|-------|------|------|-------|------|-----------|----------|")
+
# Sort providers: specific providers first, then cross-provider tests
sorted_providers = []
cross_provider = []
for p in sorted(provider_stats.keys()):
- if "Cross-Provider" in p or p == "Other Tests":
+ if 'Cross-Provider' in p or p == 'Other Tests':
cross_provider.append(p)
else:
sorted_providers.append(p)
-
+
all_providers = sorted_providers + cross_provider
-
+
for provider in all_providers:
stats = provider_stats[provider]
- total = (
- stats["passed"]
- + stats["failed"]
- + stats["errors"]
- + stats["skipped"]
- )
- pass_rate = (stats["passed"] / total * 100) if total > 0 else 0
-
- f.write(
- f"\n| {provider} | {total} | {stats['passed']} | {stats['failed']} | "
- )
+ total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
+ pass_rate = (stats['passed'] / total * 100) if total > 0 else 0
+
+ f.write(f"\n| {provider} | {total} | {stats['passed']} | {stats['failed']} | ")
f.write(f"{stats['errors']} | {stats['skipped']} | {pass_rate:.1f}% | ")
f.write(f"{format_duration(stats['time'])} |")
-
+
# Detailed test results by provider
f.write("\n\n## Detailed Test Results\n\n")
-
+
for provider in sorted_providers:
if provider_tests[provider]:
stats = provider_stats[provider]
- total = (
- stats["passed"]
- + stats["failed"]
- + stats["errors"]
- + stats["skipped"]
- )
-
+ total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
+
f.write(f"### {provider}\n\n")
f.write(f"**Summary:** {stats['passed']}/{total} passed ")
- f.write(
- f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%) "
- )
+ f.write(f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%) ")
f.write(f"in {format_duration(stats['time'])}\n\n")
-
+
# Group tests by status
tests_by_status = defaultdict(list)
for test in provider_tests[provider]:
- tests_by_status[test["status"]].append(test)
-
+ tests_by_status[test['status']].append(test)
+
# Show failed tests first (if any)
- if tests_by_status["FAILED"]:
+ if tests_by_status['FAILED']:
f.write("\nFailed Tests
\n\n")
- for test in tests_by_status["FAILED"]:
+ for test in tests_by_status['FAILED']:
f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
- if test["message"]:
+ if test['message']:
# Truncate long error messages
- msg = (
- test["message"][:200] + "..."
- if len(test["message"]) > 200
- else test["message"]
- )
+ msg = test['message'][:200] + '...' if len(test['message']) > 200 else test['message']
f.write(f" > {msg}\n")
f.write("\n \n\n")
-
+
# Show errors (if any)
- if tests_by_status["ERROR"]:
+ if tests_by_status['ERROR']:
f.write("\nError Tests
\n\n")
- for test in tests_by_status["ERROR"]:
+ for test in tests_by_status['ERROR']:
f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
f.write("\n \n\n")
-
+
# Show passed tests in collapsible section
- if tests_by_status["PASSED"]:
+ if tests_by_status['PASSED']:
f.write("\nPassed Tests
\n\n")
- for test in tests_by_status["PASSED"]:
+ for test in tests_by_status['PASSED']:
f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
f.write("\n \n\n")
-
+
# Show skipped tests (if any)
- if tests_by_status["SKIPPED"]:
+ if tests_by_status['SKIPPED']:
f.write("\nSkipped Tests
\n\n")
- for test in tests_by_status["SKIPPED"]:
+ for test in tests_by_status['SKIPPED']:
f.write(f"- `{test['name']}`\n")
f.write("\n \n\n")
-
+
# Cross-provider tests in a separate section
if cross_provider:
f.write("### Cross-Provider Tests\n\n")
for provider in cross_provider:
if provider_tests[provider]:
stats = provider_stats[provider]
- total = (
- stats["passed"]
- + stats["failed"]
- + stats["errors"]
- + stats["skipped"]
- )
-
+ total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
+
f.write(f"#### {provider}\n\n")
f.write(f"**Summary:** {stats['passed']}/{total} passed ")
- f.write(
- f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%)\n\n"
- )
-
+ f.write(f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%)\n\n")
+
# For cross-provider tests, just show counts
f.write(f"- Passed: {stats['passed']}\n")
- if stats["failed"] > 0:
+ if stats['failed'] > 0:
f.write(f"- Failed: {stats['failed']}\n")
- if stats["errors"] > 0:
+ if stats['errors'] > 0:
f.write(f"- Errors: {stats['errors']}\n")
- if stats["skipped"] > 0:
+ if stats['skipped'] > 0:
f.write(f"- Skipped: {stats['skipped']}\n")
f.write("\n")
-
+
+
print_colored(f"Report generated: {output_path}", Colors.GREEN)
-
+
except Exception as e:
print_colored(f"Error generating report: {e}", Colors.RED)
raise
-
-def run_tests(
- test_path: str = "tests/llm_translation/",
- junit_xml: str = "test-results/junit.xml",
- report_path: str = "test-results/llm_translation_report.md",
- tag: str = None,
- commit: str = None,
-) -> int:
+def run_tests(test_path: str = "tests/llm_translation/",
+ junit_xml: str = "test-results/junit.xml",
+ report_path: str = "test-results/llm_translation_report.md",
+ tag: str = None,
+ commit: str = None) -> int:
"""Run the LLM translation tests and generate report"""
-
+
# Create test results directory
os.makedirs(os.path.dirname(junit_xml), exist_ok=True)
-
+
print_colored("Starting LLM Translation Tests", Colors.BOLD + Colors.BLUE)
print_colored(f"Test directory: {test_path}", Colors.CYAN)
print_colored(f"Output: {junit_xml}", Colors.CYAN)
print()
-
+
# Run pytest
cmd = [
- "poetry",
- "run",
- "pytest",
- test_path,
+ "poetry", "run", "pytest", test_path,
f"--junitxml={junit_xml}",
"-v",
"--tb=short",
"--maxfail=500",
- "-n",
- "auto",
+ "-n", "auto"
]
-
+
# Add timeout if pytest-timeout is installed
try:
- subprocess.run(
- ["poetry", "run", "python", "-c", "import pytest_timeout"],
- capture_output=True,
- check=True,
- )
+ subprocess.run(["poetry", "run", "python", "-c", "import pytest_timeout"],
+ capture_output=True, check=True)
cmd.extend(["--timeout=300"])
except:
- print_colored(
- "Warning: pytest-timeout not installed, skipping timeout option",
- Colors.YELLOW,
- )
-
+ print_colored("Warning: pytest-timeout not installed, skipping timeout option", Colors.YELLOW)
+
print_colored("Running pytest with command:", Colors.YELLOW)
print(f" {' '.join(cmd)}")
print()
-
+
# Run the tests
result = subprocess.run(cmd, capture_output=False)
-
+
# Generate the report regardless of test outcome
if os.path.exists(junit_xml):
print()
print_colored("Generating test report...", Colors.BLUE)
generate_markdown_report(junit_xml, report_path, tag, commit)
-
+
# Print summary to console
print()
print_colored("Test Summary:", Colors.BOLD + Colors.PURPLE)
-
+
# Parse XML for quick summary
tree = ET.parse(junit_xml)
root = tree.getroot()
-
- if root.tag == "testsuites":
- suites = root.findall("testsuite")
+
+ if root.tag == 'testsuites':
+ suites = root.findall('testsuite')
else:
suites = [root]
-
- total = sum(int(s.get("tests", 0)) for s in suites)
- failures = sum(int(s.get("failures", 0)) for s in suites)
- errors = sum(int(s.get("errors", 0)) for s in suites)
- skipped = sum(int(s.get("skipped", 0)) for s in suites)
+
+ total = sum(int(s.get('tests', 0)) for s in suites)
+ failures = sum(int(s.get('failures', 0)) for s in suites)
+ errors = sum(int(s.get('errors', 0)) for s in suites)
+ skipped = sum(int(s.get('skipped', 0)) for s in suites)
passed = total - failures - errors - skipped
-
+
print(f" Total: {total}")
print_colored(f" Passed: {passed}", Colors.GREEN)
if failures > 0:
@@ -453,77 +381,59 @@ def run_tests(
print_colored(f" Errors: {errors}", Colors.RED)
if skipped > 0:
print_colored(f" Skipped: {skipped}", Colors.YELLOW)
-
+
if total > 0:
pass_rate = (passed / total) * 100
- color = (
- Colors.GREEN
- if pass_rate >= 80
- else Colors.YELLOW
- if pass_rate >= 60
- else Colors.RED
- )
+ color = Colors.GREEN if pass_rate >= 80 else Colors.YELLOW if pass_rate >= 60 else Colors.RED
print_colored(f" Pass Rate: {pass_rate:.1f}%", color)
else:
print_colored("No test results found!", Colors.RED)
-
+
print()
print_colored("Test run complete!", Colors.BOLD + Colors.GREEN)
-
+
return result.returncode
-
if __name__ == "__main__":
import argparse
-
+
parser = argparse.ArgumentParser(description="Run LLM Translation Tests")
- parser.add_argument(
- "--test-path", default="tests/llm_translation/", help="Path to test directory"
- )
- parser.add_argument(
- "--junit-xml",
- default="test-results/junit.xml",
- help="Path for JUnit XML output",
- )
- parser.add_argument(
- "--report",
- default="test-results/llm_translation_report.md",
- help="Path for markdown report",
- )
+ parser.add_argument("--test-path", default="tests/llm_translation/",
+ help="Path to test directory")
+ parser.add_argument("--junit-xml", default="test-results/junit.xml",
+ help="Path for JUnit XML output")
+ parser.add_argument("--report", default="test-results/llm_translation_report.md",
+ help="Path for markdown report")
parser.add_argument("--tag", help="Git tag or version")
parser.add_argument("--commit", help="Git commit SHA")
-
+
args = parser.parse_args()
-
+
# Get git info if not provided
if not args.commit:
try:
- result = subprocess.run(
- ["git", "rev-parse", "HEAD"], capture_output=True, text=True
- )
+ result = subprocess.run(["git", "rev-parse", "HEAD"],
+ capture_output=True, text=True)
if result.returncode == 0:
args.commit = result.stdout.strip()
except:
pass
-
+
if not args.tag:
try:
- result = subprocess.run(
- ["git", "describe", "--tags", "--abbrev=0"],
- capture_output=True,
- text=True,
- )
+ result = subprocess.run(["git", "describe", "--tags", "--abbrev=0"],
+ capture_output=True, text=True)
if result.returncode == 0:
args.tag = result.stdout.strip()
except:
pass
-
+
exit_code = run_tests(
test_path=args.test_path,
junit_xml=args.junit_xml,
report_path=args.report,
tag=args.tag,
- commit=args.commit,
+ commit=args.commit
)
-
- sys.exit(exit_code)
+
+ sys.exit(exit_code)
\ No newline at end of file
diff --git a/.github/workflows/run_observatory_tests.yml b/.github/workflows/run_observatory_tests.yml
new file mode 100644
index 00000000000..a25b96766d7
--- /dev/null
+++ b/.github/workflows/run_observatory_tests.yml
@@ -0,0 +1,229 @@
+name: Run Observatory Tests
+on:
+ workflow_dispatch:
+ inputs:
+ tag:
+ description: "Docker image tag to test (e.g. v1.61.0.rc1)"
+ required: true
+ type: string
+ commit_hash:
+ description: "Commit hash (defaults to HEAD of current branch)"
+ required: false
+ type: string
+ workflow_call:
+ inputs:
+ tag:
+ description: "Docker image tag to test"
+ required: true
+ type: string
+ commit_hash:
+ description: "Commit hash of the release"
+ required: true
+ type: string
+
+permissions:
+ contents: read
+
+env:
+ LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }}
+
+jobs:
+ observatory-tests:
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
+
+ - name: Validate tag input
+ env:
+ TAG: ${{ inputs.tag }}
+ run: |
+ if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+ echo "Invalid tag format: $TAG (expected vX.Y.Z...)"
+ exit 1
+ fi
+
+ - name: Start LiteLLM container
+ env:
+ TAG: ${{ inputs.tag }}
+ AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
+ AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
+ WORKSPACE: ${{ github.workspace }}
+ run: |
+ docker run -d \
+ --name litellm-rc \
+ -p 4000:4000 \
+ -v "${WORKSPACE}/.github/observatory/litellm_config.yaml:/app/config.yaml" \
+ -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
+ -e AZURE_API_KEY="${AZURE_API_KEY}" \
+ -e AZURE_API_BASE="${AZURE_API_BASE}" \
+ "litellm/litellm:${TAG}" \
+ --config /app/config.yaml --port 4000
+
+ - name: Wait for LiteLLM health check
+ run: |
+ echo "Waiting for LiteLLM to be ready..."
+ for i in $(seq 1 30); do
+ if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then
+ echo "LiteLLM is healthy"
+ exit 0
+ fi
+ echo "Attempt $i/30 - not ready yet, waiting 10s..."
+ sleep 10
+ done
+ echo "LiteLLM failed to start within 5 minutes"
+ docker logs litellm-rc
+ exit 1
+
+ - name: Start cloudflared tunnel
+ run: |
+ # Install cloudflared (pinned version + checksum)
+ curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared
+ echo "afdfadd1ef552e66bffc35246fe30a9bd578356d2d386de95585ccfc432472b8 /usr/local/bin/cloudflared" | sha256sum -c -
+ chmod +x /usr/local/bin/cloudflared
+
+ # Start a quick tunnel (no account needed) and capture the URL
+ cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 &
+ CLOUDFLARED_PID=$!
+ echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV
+
+ # Wait for tunnel URL to appear in logs
+ echo "Waiting for tunnel URL..."
+ for i in $(seq 1 30); do
+ TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true)
+ if [ -n "$TUNNEL_URL" ]; then
+ echo "Tunnel URL: $TUNNEL_URL"
+ echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV
+ exit 0
+ fi
+ sleep 2
+ done
+ echo "Failed to get tunnel URL"
+ cat /tmp/cloudflared.log
+ exit 1
+
+ - name: Verify tunnel connectivity
+ run: |
+ echo "Testing tunnel at ${TUNNEL_URL}..."
+ # Quick tunnels need time for DNS propagation; retry to avoid
+ # transient NXDOMAIN (curl exit code 6) on first attempt.
+ for i in $(seq 1 10); do
+ if curl -sf "${TUNNEL_URL}/health/liveliness" > /dev/null 2>&1; then
+ echo "Tunnel is working (attempt $i)"
+ exit 0
+ fi
+ echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..."
+ sleep 5
+ done
+ echo "Tunnel failed to become reachable after 50s"
+ cat /tmp/cloudflared.log
+ exit 1
+
+ - name: Trigger observatory test run
+ id: trigger
+ env:
+ OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+ OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+ run: |
+ PAYLOAD=$(jq -n \
+ --arg url "${TUNNEL_URL}" \
+ --arg key "${LITELLM_MASTER_KEY}" \
+ '{
+ deployment_url: $url,
+ api_key: $key,
+ test_suite: "TestOAIAzureRelease",
+ models: ["gpt-4o-mini", "gpt-4o"]
+ }')
+ RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \
+ -H "Content-Type: application/json" \
+ -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \
+ -d "$PAYLOAD")
+ HTTP_CODE=$(echo "$RESPONSE" | tail -1)
+ BODY=$(echo "$RESPONSE" | head -n -1)
+ echo "Response ($HTTP_CODE): $BODY"
+ if [ "$HTTP_CODE" -ge 400 ]; then
+ echo "Failed to trigger test run"
+ exit 1
+ fi
+
+ # Extract request_id for polling this specific run
+ REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id')
+ if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then
+ echo "Failed to extract request_id from response"
+ exit 1
+ fi
+ echo "Request ID: $REQUEST_ID"
+ echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT
+
+ - name: Poll for test completion
+ id: poll
+ env:
+ OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+ OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+ REQUEST_ID: ${{ steps.trigger.outputs.request_id }}
+ run: |
+ TIMEOUT=900 # 15 minutes
+ INTERVAL=30
+ ELAPSED=0
+ while [ $ELAPSED -lt $TIMEOUT ]; do
+ STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \
+ -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}")
+ RUN_STATUS=$(echo "$STATUS" | jq -r '.status')
+ echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS"
+
+ if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then
+ echo "Test finished with status: $RUN_STATUS"
+ echo "$STATUS" > /tmp/observatory_result.json
+ exit 0
+ fi
+
+ sleep $INTERVAL
+ ELAPSED=$((ELAPSED + INTERVAL))
+ done
+ echo "Timed out waiting for test to complete after ${TIMEOUT}s"
+ exit 1
+
+ - name: Verify test results
+ run: |
+ RESULT=$(cat /tmp/observatory_result.json)
+ echo "Full result: $RESULT"
+
+ STATUS=$(echo "$RESULT" | jq -r '.status')
+ TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false')
+ FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"')
+ ERROR=$(echo "$RESULT" | jq -r '.error // empty')
+
+ echo "Status: $STATUS"
+ echo "Test passed: $TEST_PASSED"
+ echo "Failure rate: $FAILURE_RATE"
+
+ if [ -n "$ERROR" ]; then
+ echo "Error: $ERROR"
+ fi
+
+ if [ "$STATUS" = "failed" ]; then
+ echo "Test run failed"
+ exit 1
+ fi
+
+ if [ "$TEST_PASSED" != "true" ]; then
+ echo "Tests did not pass (failure rate: $FAILURE_RATE)"
+ exit 1
+ fi
+
+ echo "All tests passed!"
+
+ - name: Print LiteLLM logs on failure
+ if: failure()
+ run: |
+ docker logs litellm-rc 2>/dev/null || true
+ cat /tmp/cloudflared.log 2>/dev/null || true
+
+ - name: Cleanup
+ if: always()
+ run: |
+ kill "$CLOUDFLARED_PID" 2>/dev/null || true
+ docker rm -f litellm-rc 2>/dev/null || true
diff --git a/.github/workflows/scan_duplicate_issues.yml b/.github/workflows/scan_duplicate_issues.yml
new file mode 100644
index 00000000000..222ff11f304
--- /dev/null
+++ b/.github/workflows/scan_duplicate_issues.yml
@@ -0,0 +1,48 @@
+name: Scan Duplicate Issues (One-Time)
+
+on:
+ workflow_dispatch:
+ inputs:
+ threshold:
+ description: "Similarity threshold (0-1)"
+ required: false
+ default: "0.85"
+ close:
+ description: "Actually close duplicates (false = dry run)"
+ required: false
+ type: boolean
+ default: false
+
+jobs:
+ scan:
+ runs-on: ubuntu-latest
+ permissions:
+ issues: write
+ contents: read
+ steps:
+ - name: Checkout scripts
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ sparse-checkout: .github/scripts
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.13"
+
+ - name: Scan for duplicate issues
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ INPUT_THRESHOLD: ${{ inputs.threshold }}
+ INPUT_CLOSE: ${{ inputs.close }}
+ run: |
+ CLOSE_FLAG=""
+ if [ "$INPUT_CLOSE" = "true" ]; then
+ CLOSE_FLAG="--close"
+ fi
+ python3 .github/scripts/close_duplicate_issues.py \
+ --scan \
+ --repo ${{ github.repository }} \
+ --threshold "$INPUT_THRESHOLD" \
+ $CLOSE_FLAG
diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml
new file mode 100644
index 00000000000..3a00064c3bd
--- /dev/null
+++ b/.github/workflows/scorecard.yml
@@ -0,0 +1,47 @@
+name: Scorecard supply-chain security
+
+on:
+ branch_protection_rule:
+ schedule:
+ - cron: '27 12 * * 4'
+ push:
+ branches: ["main"]
+
+permissions: read-all
+
+jobs:
+ analysis:
+ name: Scorecard analysis
+ runs-on: ubuntu-latest
+ if: github.event.repository.default_branch == github.ref_name
+ permissions:
+ security-events: write
+ id-token: write
+ # Uncomment for private repos if needed:
+ # contents: read
+ # actions: read
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ persist-credentials: false
+
+ - name: Run analysis
+ uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1
+ with:
+ results_file: results.sarif
+ results_format: sarif
+ publish_results: true
+
+ - name: Upload artifact
+ uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+ with:
+ name: SARIF file
+ path: results.sarif
+ retention-days: 5
+
+ - name: Upload to code scanning
+ uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
+ with:
+ sarif_file: results.sarif
diff --git a/.github/workflows/simple_pypi_publish.yml b/.github/workflows/simple_pypi_publish.yml
deleted file mode 100644
index e1830556819..00000000000
--- a/.github/workflows/simple_pypi_publish.yml
+++ /dev/null
@@ -1,67 +0,0 @@
-name: Simple PyPI Publish
-
-on:
- workflow_dispatch:
- inputs:
- version:
- description: 'Version to publish (e.g., 1.74.10)'
- required: true
- type: string
-
-env:
- TWINE_USERNAME: __token__
-
-jobs:
- publish:
- runs-on: ubuntu-latest
- if: github.repository == 'BerriAI/litellm'
-
- steps:
- - name: Checkout code
- uses: actions/checkout@v4
-
- - name: Set up Python
- uses: actions/setup-python@v4
- with:
- python-version: '3.8'
-
- - name: Install dependencies
- run: |
- python -m pip install --upgrade pip
- pip install toml build wheel twine
-
- - name: Update version in pyproject.toml
- run: |
- python -c "
- import toml
-
- with open('pyproject.toml', 'r') as f:
- data = toml.load(f)
-
- data['tool']['poetry']['version'] = '${{ github.event.inputs.version }}'
-
- with open('pyproject.toml', 'w') as f:
- toml.dump(data, f)
-
- print(f'Updated version to ${{ github.event.inputs.version }}')
- "
-
- - name: Copy model prices file
- run: |
- cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
-
- - name: Build package
- run: |
- rm -rf build dist
- python -m build
-
- - name: Publish to PyPI
- env:
- TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
- run: |
- twine upload dist/*
-
- - name: Output success
- run: |
- echo "✅ Successfully published litellm v${{ github.event.inputs.version }} to PyPI"
- echo "📦 Package: https://pypi.org/project/litellm/${{ github.event.inputs.version }}/"
\ No newline at end of file
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index 5a9b19fc9ca..c905bb12312 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -2,19 +2,24 @@ name: "Stale Issue Management"
on:
schedule:
- - cron: '0 0 * * *' # Runs daily at midnight UTC
+ - cron: "0 0 * * *" # Runs daily at midnight UTC
workflow_dispatch:
+permissions:
+ issues: write
+ pull-requests: write
+
jobs:
stale:
+ if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- - uses: actions/stale@v8
+ - uses: actions/stale@1160a2240286f5da8ec72b1c0816ce2481aabf84 # v8
with:
repo-token: "${{ secrets.GITHUB_TOKEN }}"
stale-issue-message: "This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs."
stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs."
- days-before-stale: 90 # Revert to 60 days
- days-before-close: 7 # Revert to 7 days
+ days-before-stale: 90 # Revert to 60 days
+ days-before-close: 7 # Revert to 7 days
stale-issue-label: "stale"
- operations-per-run: 1000
\ No newline at end of file
+ operations-per-run: 1000
diff --git a/.github/workflows/sync-schema.yml b/.github/workflows/sync-schema.yml
new file mode 100644
index 00000000000..72a5c56293e
--- /dev/null
+++ b/.github/workflows/sync-schema.yml
@@ -0,0 +1,73 @@
+name: Sync schema.prisma copies
+
+on:
+ pull_request:
+ paths:
+ - 'schema.prisma'
+
+# Scoped to ONLY the permissions needed:
+# - contents:write to push the sync commit to the PR branch
+# - pull-requests:read is implicit (needed to check out the PR)
+permissions:
+ contents: write
+
+jobs:
+ sync:
+ name: Copy root schema to proxy and proxy-extras
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ # Only run on PRs from branches in THIS repo (not forks).
+ # Fork PRs cannot push back to the head branch with GITHUB_TOKEN,
+ # and pull_request events from forks have read-only tokens anyway.
+ # Also reject PRs from branches named after protected branches to
+ # prevent pushing directly to main/master.
+ if: >-
+ github.event.pull_request.head.repo.full_name == github.repository
+ && github.head_ref != 'main'
+ && github.head_ref != 'master'
+ steps:
+ - name: Checkout PR branch by SHA
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ # Use the merge commit SHA for safety — github.head_ref is an
+ # attacker-controlled string (the branch name) and could contain
+ # unusual characters that cause unexpected git behavior.
+ ref: ${{ github.event.pull_request.head.sha }}
+ persist-credentials: true # needed for git push
+
+ - name: Reject symlinked schema files
+ run: |
+ for f in schema.prisma litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; do
+ if [ -L "$f" ]; then
+ echo "::error file=$f::$f is a symlink, which is not allowed"
+ exit 1
+ fi
+ done
+
+ - name: Copy root schema to other locations
+ run: |
+ cp schema.prisma litellm/proxy/schema.prisma
+ cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+
+ - name: Check for changes
+ id: diff
+ run: |
+ if git diff --quiet -- litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; then
+ echo "changed=false" >> "$GITHUB_OUTPUT"
+ echo "Schemas already in sync. Nothing to do."
+ else
+ echo "changed=true" >> "$GITHUB_OUTPUT"
+ echo "Schema copies need updating."
+ fi
+
+ - name: Commit synced schemas
+ if: steps.diff.outputs.changed == 'true'
+ run: |
+ # Push to the PR's head branch (need the branch name for git push).
+ # We checked out by SHA above for safety, so configure the push target explicitly.
+ git config user.name "github-actions[bot]"
+ git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+ git checkout -B "$GITHUB_HEAD_REF"
+ git add -- litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+ git commit -m "chore: sync schema.prisma copies from root"
+ git push origin "HEAD:$GITHUB_HEAD_REF"
diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml
index 7c5c269f899..5bb85716a17 100644
--- a/.github/workflows/test-linting.yml
+++ b/.github/workflows/test-linting.yml
@@ -2,7 +2,10 @@ name: LiteLLM Linting
on:
pull_request:
- branches: [ main ]
+ branches: [main]
+
+permissions:
+ contents: read
jobs:
lint:
@@ -10,67 +13,103 @@ jobs:
timeout-minutes: 5
steps:
- - uses: actions/checkout@v4
- with:
- fetch-depth: 0
- clean: true
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ fetch-depth: 0
+ clean: true
+ persist-credentials: false
- - name: Set up Python
- uses: actions/setup-python@v4
- with:
- python-version: '3.12'
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
- - name: Install Poetry
- uses: snok/install-poetry@v1
+ - name: Install Poetry
+ run: pip install 'poetry==2.3.2'
- - name: Clean Python cache
- run: |
- find . -type d -name "__pycache__" -exec rm -rf {} + || true
- find . -name "*.pyc" -delete || true
+ - name: Clean Python cache
+ run: |
+ find . -type d -name "__pycache__" -exec rm -rf {} + || true
+ find . -name "*.pyc" -delete || true
- - name: Install dependencies
- run: |
- poetry lock
- poetry install --with dev
- poetry run pip install openai==1.100.1
+ - name: Check poetry.lock is up to date
+ run: |
+ poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1)
- - name: Run Black formatting
- run: |
- cd litellm
- poetry run black .
- cd ..
+ - name: Install dependencies
+ run: |
+ poetry install --with dev
- - name: Debug - Check file state
- run: |
- echo "Current branch:"
- git branch --show-current
- echo "Last 3 commits:"
- git log --oneline -3
- echo "File content around line 43:"
- head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
-
- - name: Run Ruff linting
- run: |
- cd litellm
- poetry run ruff check .
- cd ..
+ - name: Check Black formatting
+ run: |
+ cd litellm
+ poetry run black --check --exclude '/enterprise/' .
+ cd ..
- - name: Print OpenAI version
- run: |
- poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
+ - name: Debug - Check file state
+ run: |
+ echo "Current branch:"
+ git branch --show-current
+ echo "Last 3 commits:"
+ git log --oneline -3
+ echo "File content around line 43:"
+ head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
- - name: Run MyPy type checking
- run: |
- cd litellm
- poetry run mypy .
- cd ..
+ - name: Run Ruff linting
+ run: |
+ cd litellm
+ poetry run ruff check .
+ cd ..
- - name: Check for circular imports
- run: |
- cd litellm
- poetry run python ../tests/documentation_tests/test_circular_imports.py
- cd ..
+ - name: Print OpenAI version
+ run: |
+ poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
- - name: Check import safety
- run: |
- poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+ - name: Run MyPy type checking
+ run: |
+ cd litellm
+ poetry run mypy .
+ cd ..
+
+ - name: Check for circular imports
+ run: |
+ cd litellm
+ poetry run python ../tests/documentation_tests/test_circular_imports.py
+ cd ..
+
+ - name: Check import safety
+ run: |
+ poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+
+ secret-scan:
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ permissions:
+ contents: read
+
+ steps:
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+
+ - name: Run secret scan test
+ run: |
+ pip install 'pytest==9.0.2'
+ pytest tests/litellm/test_no_hardcoded_secrets.py -v
+
+ - name: Run ggshield secret scan
+ env:
+ GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
+ run: |
+ if [ -n "$GITGUARDIAN_API_KEY" ]; then
+ pip install 'ggshield==1.48.0'
+ ggshield secret scan repo .
+ else
+ echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"
+ fi
diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml
new file mode 100644
index 00000000000..dafabe4d83e
--- /dev/null
+++ b/.github/workflows/test-litellm-matrix.yml
@@ -0,0 +1,214 @@
+name: LiteLLM Unit Tests (Matrix)
+
+on:
+ pull_request:
+ branches: [main]
+
+permissions:
+ contents: read
+
+# Cancel in-progress runs for the same PR
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ timeout-minutes: 20 # Increased from 15 to 20
+ strategy:
+ fail-fast: false
+ matrix:
+ test-group:
+ # tests/test_litellm split by subdirectory (~560 files total)
+ # Vertex AI tests separated for better isolation (prevent auth/env pollution)
+ - name: "llms-vertex"
+ path: "tests/test_litellm/llms/vertex_ai"
+ workers: 1
+ reruns: 2
+ - name: "llms-other"
+ path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
+ workers: 2
+ reruns: 2
+ # tests/test_litellm/proxy split by subdirectory (~180 files total)
+ - name: "proxy-guardrails"
+ path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers"
+ workers: 2
+ reruns: 2
+ - name: "proxy-core"
+ path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine"
+ workers: 2
+ reruns: 2
+ - name: "proxy-misc"
+ path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py"
+ workers: 2
+ reruns: 2
+ - name: "integrations"
+ path: "tests/test_litellm/integrations"
+ workers: 2
+ reruns: 3 # Integration tests tend to be flakier
+ - name: "core-utils"
+ path: "tests/test_litellm/litellm_core_utils"
+ workers: 2
+ reruns: 1
+ - name: "other-1"
+ # responses (5942) + caching (1723) + types (819) ≈ 8.5k lines
+ path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
+ workers: 2
+ reruns: 2
+ - name: "other-2"
+ # enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines
+ path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils"
+ workers: 2
+ reruns: 2
+ - name: "other-3"
+ # remaining dirs ≈ 8.0k lines
+ path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores"
+ workers: 2
+ reruns: 2
+ - name: "root"
+ path: "tests/test_litellm/test_*.py"
+ workers: 2
+ reruns: 2
+ # tests/proxy_unit_tests split alphabetically (~48 files total)
+ - name: "proxy-unit-a1"
+ # test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest
+ path: "tests/proxy_unit_tests/test_[a-j]*.py"
+ workers: 2
+ reruns: 1
+ - name: "proxy-unit-a2"
+ # test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback
+ path: "tests/proxy_unit_tests/test_[k-o]*.py"
+ workers: 2
+ reruns: 1
+ - name: "proxy-unit-b1"
+ # lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*)
+ path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
+ workers: 2
+ reruns: 1
+ - name: "proxy-unit-b2"
+ # proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests
+ path: "tests/proxy_unit_tests/test_proxy_server.py"
+ workers: 2
+ reruns: 1
+ - name: "proxy-unit-b3"
+ # proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests
+ path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
+ workers: 2
+ reruns: 1
+ - name: "proxy-unit-b4"
+ # proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter
+ path: "tests/proxy_unit_tests/test_proxy_utils.py"
+ workers: 2
+ reruns: 1
+ - name: "proxy-unit-b5"
+ # proxy_token_counter (1279) - runs independently from utils
+ path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
+ workers: 2
+ reruns: 1
+ - name: "proxy-unit-b6"
+ # test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62)
+ path: "tests/proxy_unit_tests/test_[r-t]*.py"
+ workers: 2
+ reruns: 1
+ - name: "proxy-unit-b7"
+ # test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157)
+ path: "tests/proxy_unit_tests/test_[u-z]*.py"
+ workers: 2
+ reruns: 1
+
+ name: test (${{ matrix.test-group.name }})
+
+ steps:
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+
+ - name: Install Poetry
+ run: pip install 'poetry==2.3.2'
+
+ - name: Cache Poetry dependencies
+ uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0
+ with:
+ path: |
+ ~/.cache/pypoetry
+ ~/.cache/pip
+ .venv
+ key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-poetry-
+
+ - name: Install dependencies
+ run: |
+ poetry config virtualenvs.in-project true
+ poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+ # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies
+ poetry run pip install google-genai==1.22.0 \
+ google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
+
+ - name: Setup litellm-enterprise
+ run: |
+ poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+ - name: Generate Prisma client
+ env:
+ PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+ run: |
+ poetry run pip install nodejs-wheel-binaries==24.13.1
+ poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+ - name: Run tests - ${{ matrix.test-group.name }}
+ run: |
+ poetry run pytest ${{ matrix.test-group.path }} \
+ --tb=short -vv \
+ --maxfail=10 \
+ -n ${{ matrix.test-group.workers }} \
+ --reruns ${{ matrix.test-group.reruns }} \
+ --reruns-delay 1 \
+ --dist=loadscope \
+ --durations=20 \
+ --cov=litellm \
+ --cov-report=xml:coverage-${{ matrix.test-group.name }}.xml \
+ --cov-config=pyproject.toml
+
+ - name: Save coverage report
+ if: always()
+ uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+ with:
+ name: coverage-${{ matrix.test-group.name }}
+ path: coverage-${{ matrix.test-group.name }}.xml
+ retention-days: 1
+
+ upload-coverage:
+ name: Upload coverage to Codecov
+ needs: test
+ if: always()
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ id-token: write # Required for OIDC tokenless upload
+ pull-requests: write # Required for Codecov PR comments
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+
+ - name: Download all coverage reports
+ uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
+ with:
+ pattern: coverage-*
+ path: coverage-reports
+ merge-multiple: true
+
+ - name: Upload to Codecov
+ uses: codecov/codecov-action@aa56896cf108bd10b5eb883cd1d24196da57f695 # v5.5.4
+ with:
+ use_oidc: true
+ directory: coverage-reports
+ root_dir: ${{ github.workspace }}
+ fail_ci_if_error: false
diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml
new file mode 100644
index 00000000000..bef568298e0
--- /dev/null
+++ b/.github/workflows/test-litellm-ui-build.yml
@@ -0,0 +1,34 @@
+name: UI Build Check
+permissions:
+ contents: read
+
+on:
+ pull_request:
+ branches: [main]
+
+jobs:
+ build-ui:
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ defaults:
+ run:
+ working-directory: ui/litellm-dashboard
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
+
+ - name: Setup Node.js
+ uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0
+ with:
+ node-version: "20"
+ cache: "npm"
+ cache-dependency-path: ui/litellm-dashboard/package-lock.json
+
+ - name: Install dependencies
+ run: npm ci
+
+ - name: Build
+ run: npm run build
diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml
index d9cf2e74a11..0c040b3ebe7 100644
--- a/.github/workflows/test-litellm.yml
+++ b/.github/workflows/test-litellm.yml
@@ -1,8 +1,15 @@
name: LiteLLM Mock Tests (folder - tests/test_litellm)
+# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs
+# the same tests in parallel across 10 jobs for faster CI times.
+# Kept for manual debugging only.
on:
- pull_request:
- branches: [ main ]
+ workflow_dispatch: # Manual trigger only
+ # pull_request:
+ # branches: [ main ]
+
+permissions:
+ contents: read
jobs:
test:
@@ -10,37 +17,37 @@ jobs:
timeout-minutes: 25
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
- - name: Thank You Message
- run: |
- echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
- echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
+ - name: Thank You Message
+ run: |
+ echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
+ echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
- - name: Set up Python
- uses: actions/setup-python@v4
- with:
- python-version: '3.12'
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
- - name: Install Poetry
- uses: snok/install-poetry@v1
+ - name: Install Poetry
+ run: pip install 'poetry==2.3.2'
- - name: Install dependencies
- run: |
- poetry lock
- poetry install --with dev,proxy-dev --extras "proxy semantic-router"
- poetry run pip install "pytest-retry==1.6.3"
- poetry run pip install pytest-xdist
- poetry run pip install "google-genai==1.22.0"
- poetry run pip install "google-cloud-aiplatform>=1.38"
- poetry run pip install "fastapi-offline==1.7.3"
- poetry run pip install "python-multipart==0.0.22"
- poetry run pip install "openapi-core"
- - name: Setup litellm-enterprise as local package
- run: |
- cd enterprise
- poetry run pip install -e .
- cd ..
- - name: Run tests
- run: |
- poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50
+ - name: Install dependencies
+ run: |
+ poetry lock
+ poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+ poetry run pip install "pytest-retry==1.6.3"
+ poetry run pip install 'pytest-xdist==3.8.0'
+ poetry run pip install "google-genai==1.22.0"
+ poetry run pip install "google-cloud-aiplatform==1.115.0"
+ poetry run pip install "fastapi-offline==1.7.3"
+ poetry run pip install "python-multipart==0.0.22"
+ poetry run pip install "openapi-core==0.23.0"
+ - name: Setup litellm-enterprise as local package
+ run: |
+ poetry run pip install --force-reinstall --no-deps -e enterprise/
+ - name: Run tests
+ run: |
+ poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50
diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml
index e19e67c9c4f..1b228ab76bb 100644
--- a/.github/workflows/test-mcp.yml
+++ b/.github/workflows/test-mcp.yml
@@ -2,7 +2,10 @@ name: LiteLLM MCP Tests (folder - tests/mcp_tests)
on:
pull_request:
- branches: [ main ]
+ branches: [main]
+
+permissions:
+ contents: read
jobs:
test:
@@ -10,40 +13,40 @@ jobs:
timeout-minutes: 25
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
- - name: Thank You Message
- run: |
- echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
- echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
+ - name: Thank You Message
+ run: |
+ echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
+ echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
- - name: Set up Python
- uses: actions/setup-python@v4
- with:
- python-version: '3.12'
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
- - name: Install Poetry
- uses: snok/install-poetry@v1
+ - name: Install Poetry
+ run: pip install 'poetry==2.3.2'
- - name: Install dependencies
- run: |
- poetry lock
- poetry install --with dev,proxy-dev --extras "proxy semantic-router"
- poetry run pip install "pytest==7.3.1"
- poetry run pip install "pytest-retry==1.6.3"
- poetry run pip install "pytest-cov==5.0.0"
- poetry run pip install "pytest-asyncio==0.21.1"
- poetry run pip install "respx==0.22.0"
- poetry run pip install "pydantic==2.11.0"
- poetry run pip install "mcp==1.25.0"
- poetry run pip install pytest-xdist
+ - name: Install dependencies
+ run: |
+ poetry lock
+ poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+ poetry run pip install "pytest==7.3.1"
+ poetry run pip install "pytest-retry==1.6.3"
+ poetry run pip install "pytest-cov==5.0.0"
+ poetry run pip install "pytest-asyncio==0.21.1"
+ poetry run pip install "respx==0.22.0"
+ poetry run pip install "pydantic==2.11.0"
+ poetry run pip install "mcp==1.25.0"
+ poetry run pip install 'pytest-xdist==3.8.0'
- - name: Setup litellm-enterprise as local package
- run: |
- cd enterprise
- python -m pip install -e .
- cd ..
+ - name: Setup litellm-enterprise as local package
+ run: |
+ poetry run pip install --force-reinstall --no-deps -e enterprise/
- - name: Run MCP tests
- run: |
- poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5
+ - name: Run MCP tests
+ run: |
+ poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5
diff --git a/.github/workflows/test-model-map.yaml b/.github/workflows/test-model-map.yaml
index ae5ac402e23..429f9e1ce0a 100644
--- a/.github/workflows/test-model-map.yaml
+++ b/.github/workflows/test-model-map.yaml
@@ -2,13 +2,18 @@ name: Validate model_prices_and_context_window.json
on:
pull_request:
- branches: [ main ]
+ branches: [main]
+
+permissions:
+ contents: read
jobs:
validate-model-prices-json:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
- name: Validate model_prices_and_context_window.json
run: |
diff --git a/.github/workflows/test-proxy-e2e-azure-batches.yml b/.github/workflows/test-proxy-e2e-azure-batches.yml
new file mode 100644
index 00000000000..7cbbe0b338f
--- /dev/null
+++ b/.github/workflows/test-proxy-e2e-azure-batches.yml
@@ -0,0 +1,97 @@
+name: Proxy E2E Azure Batches Tests
+
+on:
+ pull_request:
+ branches: [main]
+ workflow_dispatch:
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+permissions:
+ contents: read
+
+jobs:
+ proxy_e2e_azure_batches_tests:
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+
+ services:
+ postgres:
+ image: postgres:15
+ env:
+ POSTGRES_USER: llmproxy
+ POSTGRES_PASSWORD: dbpassword9090
+ POSTGRES_DB: litellm
+ ports:
+ - 5432:5432
+ options: >-
+ --health-cmd pg_isready
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+
+ steps:
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+
+ - name: Install Poetry
+ run: pip install 'poetry==2.3.2'
+
+ - name: Cache Poetry dependencies
+ uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0
+ with:
+ path: |
+ ~/.cache/pypoetry
+ ~/.cache/pip
+ .venv
+ key: ${{ runner.os }}-poetry-e2e-batches-${{ hashFiles('poetry.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-poetry-e2e-batches-
+ ${{ runner.os }}-poetry-
+
+ - name: Install dependencies
+ run: |
+ poetry config virtualenvs.in-project true
+ poetry install --with dev,proxy-dev --extras "proxy"
+ poetry run pip install psycopg2-binary==2.9.11 uvicorn==0.42.0 fastapi==0.135.2 httpx==0.28.1 tenacity==9.1.4
+
+ - name: Setup litellm-enterprise
+ run: |
+ poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+ - name: Generate Prisma client
+ env:
+ PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+ run: |
+ poetry run pip install nodejs-wheel-binaries==24.13.1
+ poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+ - name: Run Prisma migrations
+ env:
+ DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+ run: |
+ cd litellm/proxy
+ poetry run prisma migrate deploy --schema schema.prisma
+ cd ../..
+
+ - name: Run Azure Batch E2E Tests
+ env:
+ DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+ USE_LOCAL_LITELLM: "true"
+ USE_MOCK_MODELS: "true"
+ USE_STATE_TRACKER: "true"
+ LITELLM_LOG: DEBUG
+ run: |
+ poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
+ -vv -s -k "test_e2e_managed_batch" \
+ --tb=short \
+ --maxfail=3 \
+ --durations=10
diff --git a/.github/workflows/test-unit-caching-redis.yml b/.github/workflows/test-unit-caching-redis.yml
new file mode 100644
index 00000000000..ca274324f2f
--- /dev/null
+++ b/.github/workflows/test-unit-caching-redis.yml
@@ -0,0 +1,38 @@
+name: "Unit Tests: Caching (Redis)"
+
+# Uses cloud Redis credentials — only runs on trusted branches, not PRs.
+# This prevents external PRs from accessing Redis credentials.
+on:
+ push:
+ branches: [main, "litellm_*"]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ caching-redis:
+ uses: ./.github/workflows/_test-unit-services-base.yml
+ with:
+ # Redis-only tests that do NOT require provider API keys.
+ # Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py,
+ # test_router_caching.py) are in Phase 3 integration workflows.
+ test-path: >-
+ tests/local_testing/test_dual_cache.py
+ tests/local_testing/test_redis_batch_optimizations.py
+ tests/local_testing/test_router_utils.py
+ workers: 2
+ reruns: 2
+ timeout-minutes: 20
+ enable-redis: true
+ enable-postgres: false
+ secrets:
+ REDIS_HOST: ${{ secrets.REDIS_HOST }}
+ REDIS_PORT: ${{ secrets.REDIS_PORT }}
+ REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
+ DATABASE_URL: ${{ secrets.DATABASE_URL }}
+ POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+ POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
diff --git a/.github/workflows/test-unit-core-utils.yml b/.github/workflows/test-unit-core-utils.yml
new file mode 100644
index 00000000000..2f3698fdf60
--- /dev/null
+++ b/.github/workflows/test-unit-core-utils.yml
@@ -0,0 +1,20 @@
+name: "Unit Tests: Core Utilities"
+
+on:
+ pull_request:
+ branches: [main]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ core-utils:
+ uses: ./.github/workflows/_test-unit-base.yml
+ with:
+ test-path: "tests/test_litellm/litellm_core_utils"
+ workers: 2
+ reruns: 1
diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml
new file mode 100644
index 00000000000..d8b30de6844
--- /dev/null
+++ b/.github/workflows/test-unit-documentation.yml
@@ -0,0 +1,67 @@
+name: "Unit Tests: Documentation Validation"
+
+on:
+ pull_request:
+ branches: [main]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ documentation:
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+
+ steps:
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+
+ - name: Install Poetry
+ run: pip install 'poetry==2.3.2'
+
+ - name: Cache Poetry dependencies
+ uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+ with:
+ path: |
+ ~/.cache/pypoetry
+ ~/.cache/pip
+ .venv
+ key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-poetry-
+
+ - name: Install dependencies
+ run: |
+ poetry config virtualenvs.in-project true
+ poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+ poetry run pip install google-genai==1.22.0 \
+ google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
+
+ - name: Setup litellm-enterprise
+ run: |
+ poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+ - name: Generate Prisma client
+ env:
+ PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+ run: |
+ poetry run pip install nodejs-wheel-binaries==24.13.1
+ poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+ # Run the same documentation tests that CircleCI ran (as direct Python scripts)
+ - name: Run documentation validation tests
+ run: |
+ poetry run python ./tests/documentation_tests/test_env_keys.py
+ poetry run python ./tests/documentation_tests/test_router_settings.py
+ poetry run python ./tests/documentation_tests/test_api_docs.py
+ poetry run python ./tests/documentation_tests/test_circular_imports.py
diff --git a/.github/workflows/test-unit-enterprise-routing.yml b/.github/workflows/test-unit-enterprise-routing.yml
new file mode 100644
index 00000000000..13ae3efedba
--- /dev/null
+++ b/.github/workflows/test-unit-enterprise-routing.yml
@@ -0,0 +1,24 @@
+name: "Unit Tests: Enterprise, Google GenAI & Routing"
+
+on:
+ pull_request:
+ branches: [main]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ enterprise-routing:
+ uses: ./.github/workflows/_test-unit-base.yml
+ with:
+ test-path: >-
+ tests/test_litellm/enterprise
+ tests/test_litellm/google_genai
+ tests/test_litellm/router_utils
+ tests/test_litellm/router_strategy
+ workers: 2
+ reruns: 2
diff --git a/.github/workflows/test-unit-integrations.yml b/.github/workflows/test-unit-integrations.yml
new file mode 100644
index 00000000000..2789f99d81c
--- /dev/null
+++ b/.github/workflows/test-unit-integrations.yml
@@ -0,0 +1,20 @@
+name: "Unit Tests: Integrations (Callbacks & Logging)"
+
+on:
+ pull_request:
+ branches: [main]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ integrations:
+ uses: ./.github/workflows/_test-unit-base.yml
+ with:
+ test-path: "tests/test_litellm/integrations"
+ workers: 2
+ reruns: 3
diff --git a/.github/workflows/test-unit-llm-providers.yml b/.github/workflows/test-unit-llm-providers.yml
new file mode 100644
index 00000000000..6c00272b0c8
--- /dev/null
+++ b/.github/workflows/test-unit-llm-providers.yml
@@ -0,0 +1,29 @@
+name: "Unit Tests: LLM Provider Transformations"
+
+on:
+ pull_request:
+ branches: [main]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ vertex-ai:
+ name: Vertex AI
+ uses: ./.github/workflows/_test-unit-base.yml
+ with:
+ test-path: "tests/test_litellm/llms/vertex_ai"
+ workers: 1
+ reruns: 2
+
+ other-providers:
+ name: All Other Providers
+ uses: ./.github/workflows/_test-unit-base.yml
+ with:
+ test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
+ workers: 2
+ reruns: 2
diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml
new file mode 100644
index 00000000000..9228decd7cc
--- /dev/null
+++ b/.github/workflows/test-unit-misc.yml
@@ -0,0 +1,31 @@
+name: "Unit Tests: MCP, Secrets, Containers & Misc"
+
+on:
+ pull_request:
+ branches: [main]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ misc:
+ uses: ./.github/workflows/_test-unit-base.yml
+ with:
+ test-path: >-
+ tests/test_litellm/secret_managers
+ tests/test_litellm/a2a_protocol
+ tests/test_litellm/anthropic_interface
+ tests/test_litellm/completion_extras
+ tests/test_litellm/containers
+ tests/test_litellm/experimental_mcp_client
+ tests/test_litellm/images
+ tests/test_litellm/interactions
+ tests/test_litellm/passthrough
+ tests/test_litellm/vector_stores
+ tests/test_litellm/test_*.py
+ workers: 2
+ reruns: 2
diff --git a/.github/workflows/test-unit-proxy-auth.yml b/.github/workflows/test-unit-proxy-auth.yml
new file mode 100644
index 00000000000..e71821db701
--- /dev/null
+++ b/.github/workflows/test-unit-proxy-auth.yml
@@ -0,0 +1,20 @@
+name: "Unit Tests: Proxy Auth & Key Management"
+
+on:
+ pull_request:
+ branches: [main]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ proxy-auth:
+ uses: ./.github/workflows/_test-unit-base.yml
+ with:
+ test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client"
+ workers: 2
+ reruns: 2
diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml
new file mode 100644
index 00000000000..bdfb6efeef1
--- /dev/null
+++ b/.github/workflows/test-unit-proxy-db.yml
@@ -0,0 +1,45 @@
+name: "Unit Tests: Proxy DB Operations"
+
+# Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
+on:
+ push:
+ branches: [main, "litellm_*"]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ proxy-db:
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ # Key generation tests must NOT run in parallel (event loop conflicts with logging worker)
+ - test-group: key-generation
+ test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py"
+ workers: 0
+ timeout: 30
+ - test-group: auth-checks
+ test-path: "tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py"
+ workers: 8
+ timeout: 20
+ - test-group: remaining
+ test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py"
+ workers: 8
+ timeout: 20
+ uses: ./.github/workflows/_test-unit-services-base.yml
+ with:
+ test-path: ${{ matrix.test-path }}
+ workers: ${{ matrix.workers }}
+ reruns: 2
+ timeout-minutes: ${{ matrix.timeout }}
+ enable-redis: false
+ enable-postgres: true
+ secrets:
+ DATABASE_URL: ${{ secrets.DATABASE_URL }}
+ POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+ POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml
new file mode 100644
index 00000000000..caff3b3ae06
--- /dev/null
+++ b/.github/workflows/test-unit-proxy-endpoints.yml
@@ -0,0 +1,35 @@
+name: "Unit Tests: Proxy API Endpoints"
+
+on:
+ pull_request:
+ branches: [main]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ proxy-endpoints:
+ uses: ./.github/workflows/_test-unit-base.yml
+ with:
+ test-path: >-
+ tests/test_litellm/proxy/management_endpoints
+ tests/test_litellm/proxy/guardrails
+ tests/test_litellm/proxy/management_helpers
+ tests/test_litellm/proxy/anthropic_endpoints
+ tests/test_litellm/proxy/google_endpoints
+ tests/test_litellm/proxy/openai_files_endpoint
+ tests/test_litellm/proxy/response_api_endpoints
+ tests/test_litellm/proxy/image_endpoints
+ tests/test_litellm/proxy/vector_store_endpoints
+ tests/test_litellm/proxy/agent_endpoints
+ tests/test_litellm/proxy/discovery_endpoints
+ tests/test_litellm/proxy/health_endpoints
+ tests/test_litellm/proxy/public_endpoints
+ tests/test_litellm/proxy/prompts
+ tests/test_litellm/proxy/ui_crud_endpoints
+ workers: 2
+ reruns: 2
diff --git a/.github/workflows/test-unit-proxy-infra.yml b/.github/workflows/test-unit-proxy-infra.yml
new file mode 100644
index 00000000000..4dfbbe317ed
--- /dev/null
+++ b/.github/workflows/test-unit-proxy-infra.yml
@@ -0,0 +1,28 @@
+name: "Unit Tests: Proxy Infrastructure"
+
+on:
+ pull_request:
+ branches: [main]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ proxy-infra:
+ uses: ./.github/workflows/_test-unit-base.yml
+ with:
+ test-path: >-
+ tests/test_litellm/proxy/db
+ tests/test_litellm/proxy/middleware
+ tests/test_litellm/proxy/spend_tracking
+ tests/test_litellm/proxy/pass_through_endpoints
+ tests/test_litellm/proxy/_experimental
+ tests/test_litellm/proxy/experimental
+ tests/test_litellm/proxy/common_utils
+ tests/test_litellm/proxy/test_*.py
+ workers: 2
+ reruns: 2
diff --git a/.github/workflows/test-unit-proxy-legacy.yml b/.github/workflows/test-unit-proxy-legacy.yml
new file mode 100644
index 00000000000..a9391137263
--- /dev/null
+++ b/.github/workflows/test-unit-proxy-legacy.yml
@@ -0,0 +1,96 @@
+name: "Unit Tests: Proxy Legacy Tests"
+
+on:
+ pull_request:
+ branches: [main]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ strategy:
+ fail-fast: false
+ matrix:
+ test-group:
+ - name: "auth-and-jwt"
+ path: "tests/proxy_unit_tests/test_[a-j]*.py"
+ - name: "key-generation"
+ path: "tests/proxy_unit_tests/test_[k-o]*.py"
+ - name: "proxy-config"
+ path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
+ - name: "proxy-server"
+ path: "tests/proxy_unit_tests/test_proxy_server.py"
+ - name: "proxy-server-extras"
+ path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
+ - name: "proxy-utils"
+ path: "tests/proxy_unit_tests/test_proxy_utils.py"
+ - name: "proxy-token-counter"
+ path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
+ - name: "proxy-response-and-misc"
+ path: "tests/proxy_unit_tests/test_[r-t]*.py"
+ - name: "proxy-user-auth-and-spend"
+ path: "tests/proxy_unit_tests/test_[u-z]*.py"
+
+ name: ${{ matrix.test-group.name }}
+
+ steps:
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+
+ - name: Install Poetry
+ run: pip install 'poetry==2.3.2'
+
+ - name: Cache Poetry dependencies
+ uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+ with:
+ path: |
+ ~/.cache/pypoetry
+ ~/.cache/pip
+ .venv
+ key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-poetry-
+
+ - name: Install dependencies
+ run: |
+ poetry config virtualenvs.in-project true
+ poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+ poetry run pip install google-genai==1.22.0 \
+ google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
+
+ - name: Setup litellm-enterprise
+ run: |
+ poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+ - name: Generate Prisma client
+ env:
+ PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+ run: |
+ poetry run pip install nodejs-wheel-binaries==24.13.1
+ poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+ - name: Run tests - ${{ matrix.test-group.name }}
+ env:
+ TEST_PATH: ${{ matrix.test-group.path }}
+ run: |
+ poetry run pytest ${TEST_PATH} \
+ --tb=short -vv \
+ --maxfail=10 \
+ -n 2 \
+ --reruns 1 \
+ --reruns-delay 1 \
+ --dist=loadscope \
+ --durations=20
diff --git a/.github/workflows/test-unit-responses-caching-types.yml b/.github/workflows/test-unit-responses-caching-types.yml
new file mode 100644
index 00000000000..7f3acac2803
--- /dev/null
+++ b/.github/workflows/test-unit-responses-caching-types.yml
@@ -0,0 +1,20 @@
+name: "Unit Tests: Responses, Caching & Types"
+
+on:
+ pull_request:
+ branches: [main]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ responses-caching-types:
+ uses: ./.github/workflows/_test-unit-base.yml
+ with:
+ test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
+ workers: 2
+ reruns: 2
diff --git a/.github/workflows/test-unit-security.yml b/.github/workflows/test-unit-security.yml
new file mode 100644
index 00000000000..b38c82b1c24
--- /dev/null
+++ b/.github/workflows/test-unit-security.yml
@@ -0,0 +1,28 @@
+name: "Unit Tests: Security"
+
+# Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
+on:
+ push:
+ branches: [main, "litellm_*"]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ security:
+ uses: ./.github/workflows/_test-unit-services-base.yml
+ with:
+ test-path: "tests/proxy_security_tests/"
+ workers: 1
+ reruns: 2
+ timeout-minutes: 20
+ enable-redis: false
+ enable-postgres: true
+ secrets:
+ DATABASE_URL: ${{ secrets.DATABASE_URL }}
+ POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+ POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
diff --git a/.github/workflows/test_server_root_path.yml b/.github/workflows/test_server_root_path.yml
new file mode 100644
index 00000000000..47636ce8e92
--- /dev/null
+++ b/.github/workflows/test_server_root_path.yml
@@ -0,0 +1,98 @@
+name: Test Proxy SERVER_ROOT_PATH Routing
+permissions:
+ contents: read
+
+on:
+ pull_request:
+ branches: [main]
+
+jobs:
+ test-server-root-path:
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+
+ strategy:
+ matrix:
+ root_path: ["/api/v1", "/llmproxy"]
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12
+
+ - name: Build Docker image
+ uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 #v6.14
+ with:
+ context: .
+ file: ./docker/Dockerfile.non_root
+ tags: litellm-test:${{ github.sha }}
+ load: true
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
+
+ - name: Start LiteLLM container with SERVER_ROOT_PATH
+ run: |
+ docker run -d \
+ --name litellm-test \
+ -p 4000:4000 \
+ -e SERVER_ROOT_PATH="${{ matrix.root_path }}" \
+ -e LITELLM_MASTER_KEY="sk-1234" \
+ litellm-test:${{ github.sha }} \
+ --detailed_debug
+
+ - name: Wait for container to be healthy
+ run: |
+ echo "Waiting for LiteLLM to start..."
+ max_attempts=30
+ attempt=0
+
+ while [ $attempt -lt $max_attempts ]; do
+ if docker logs litellm-test 2>&1 | grep -q "Uvicorn running"; then
+ echo "LiteLLM started successfully"
+ break
+ fi
+ attempt=$((attempt + 1))
+ echo "Attempt $attempt/$max_attempts - waiting for server to start..."
+ sleep 2
+ done
+
+ if [ $attempt -eq $max_attempts ]; then
+ echo "Server failed to start within timeout"
+ docker logs litellm-test
+ exit 1
+ fi
+
+ sleep 5
+
+ - name: Show container logs
+ if: always()
+ run: docker logs litellm-test
+
+ - name: Test UI endpoint with root path
+ run: |
+ ROOT_PATH="${{ matrix.root_path }}"
+ echo "Testing UI at: http://localhost:4000${ROOT_PATH}/ui/"
+
+ for i in 1 2 3; do
+ content=$(curl -sL --max-time 5 -H "Authorization: Bearer sk-1234" "http://localhost:4000${ROOT_PATH}/ui/")
+ if echo "$content" | grep -q -E "(html| /dev/null; then
-# install_ggshield
-# fi
-
-# # Check if GITGUARDIAN_API_KEY is set (required for CI/CD)
-# if [ -z "$GITGUARDIAN_API_KEY" ]; then
-# echo "Warning: GITGUARDIAN_API_KEY environment variable is not set."
-# echo "ggshield requires a GitGuardian API key to scan for secrets."
-# echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables."
-# exit 1
-# fi
-
-# echo "Scanning codebase for secrets..."
-# echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)"
-# echo "ggshield will automatically handle rate limits and retry as needed."
-# echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml"
-
-# # Use --recursive for directory scanning and auto-confirm if prompted
-# # .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
-# # GITGUARDIAN_API_KEY environment variable will be used for authentication
-# echo y | ggshield secret scan path . --recursive || {
-# echo ""
-# echo "=========================================="
-# echo "ERROR: Secret Detection Failed"
-# echo "=========================================="
-# echo "ggshield has detected secrets in the codebase."
-# echo "Please review discovered secrets above, revoke any actively used secrets"
-# echo "from underlying systems and make changes to inject secrets dynamically at runtime."
-# echo ""
-# echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
-# echo "=========================================="
-# echo ""
-# exit 1
-# }
-
-# echo "Secret detection scans completed successfully"
-# }
-
-# Function to run Trivy scans
-run_trivy_scans() {
- echo "Running Trivy scans..."
-
- echo "Scanning LiteLLM Docs..."
- trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
-
- echo "Scanning LiteLLM UI..."
- trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
-
- echo "Trivy scans completed successfully"
-}
-
-# Function to build and scan Docker images with Grype
-run_grype_scans() {
- echo "Running Grype scans..."
-
- # Temporarily add wheel files to .dockerignore for security scans
- echo "Temporarily modifying .dockerignore to exclude problematic wheel files..."
- cp .dockerignore .dockerignore.backup 2>/dev/null || touch .dockerignore.backup
- echo "/*.whl" >> .dockerignore
-
- # Build and scan Dockerfile.database
- echo "Building and scanning Dockerfile.database..."
- docker build --no-cache -t litellm-database:latest -f ./docker/Dockerfile.database .
- grype litellm-database:latest --config ci_cd/.grype.yaml --fail-on critical
-
- # Build and scan main Dockerfile
- echo "Building and scanning main Dockerfile..."
- docker build --no-cache -t litellm:latest .
- grype litellm:latest --config ci_cd/.grype.yaml --fail-on critical
-
- # Restore original .dockerignore
- echo "Restoring original .dockerignore..."
- mv .dockerignore.backup .dockerignore
-
- # Scan the locally built LiteLLM image for vulnerabilities with CVSS >= 4.0
- echo "Scanning locally built LiteLLM image for high-severity vulnerabilities..."
- echo "Using locally built image: litellm:latest"
-
- # Allowlist of CVEs to be ignored in failure threshold/reporting
- # - CVE-2025-8869: Not applicable on Python >=3.13 (PEP 706 implemented); pip fallback unused; no OS-level fix
- # - GHSA-4xh5-x5gv-qwph: GitHub Security Advisory alias for CVE-2025-8869
- # - GHSA-5j98-mcp5-4vw2: glob CLI command injection via -c/--cmd; glob CLI is not used in the litellm runtime image,
- # and the vulnerable versions are pulled in only via OS-level/node tooling outside of our application code
- ALLOWED_CVES=(
- "CVE-2025-8869"
- "GHSA-4xh5-x5gv-qwph"
- "CVE-2025-8291" # no fix available as of Oct 11, 2025
- "GHSA-5j98-mcp5-4vw2"
- "CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image
- "CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image
- "CVE-2025-60876" # BusyBox wget HTTP request splitting - no fix available in Chainguard Wolfi base image
- "CVE-2026-0861" # Wolfi glibc still flagged even on 2.42-r5; upstream patched build unavailable yet
- "CVE-2010-4756" # glibc glob DoS - awaiting patched Wolfi glibc build
- "CVE-2019-1010022" # glibc stack guard bypass - awaiting patched Wolfi glibc build
- "CVE-2019-1010023" # glibc ldd remap issue - awaiting patched Wolfi glibc build
- "CVE-2019-1010024" # glibc ASLR mitigation bypass - awaiting patched Wolfi glibc build
- "CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build
- "CVE-2026-22184" # zlib untgz buffer overflow - untgz unused + no fixed Wolfi build yet
- "GHSA-58pv-8j8x-9vj2" # jaraco.context path traversal - setuptools vendored only (v5.3.0), not used in application code (using v6.1.0+)
- "GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code
- "GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
- "GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
- "CVE-2025-59465" # We do not use Node in application runtime, only used for building Admin UI
- "CVE-2025-55131" # We do not use Node in application runtime, only used for building Admin UI
- "CVE-2025-59466" # We do not use Node in application runtime, only used for building Admin UI
- "CVE-2025-55130" # We do not use Node in application runtime, only used for building Admin UI
- "CVE-2025-59467" # We do not use Node in application runtime, only used for building Admin UI
- "CVE-2026-21637" # We do not use Node in application runtime, only used for building Admin UI
- "CVE-2025-15281" # No fix available yet
- "CVE-2026-0865" # No fix available yet
- "CVE-2025-15282" # No fix available yet
- "CVE-2026-0672" # No fix available yet
- "CVE-2025-15366" # No fix available yet
- "CVE-2025-15367" # No fix available yet
- "CVE-2025-12781" # No fix available yet
- "CVE-2025-11468" # No fix available yet
- "CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
- )
-
- # Build JSON array of allowlisted CVE IDs for jq
- ALLOWED_IDS_JSON=$(printf '%s\n' "${ALLOWED_CVES[@]}" | jq -R . | jq -s .)
-
- echo "Checking for vulnerabilities with CVSS score >= 4.0..."
- echo "Allowlisted CVEs (ignored in threshold): ${ALLOWED_CVES[*]}"
- echo ""
-
- # Show all high-severity vulnerabilities for transparency
- TOTAL_HIGH_SEVERITY=$(grype litellm:latest -o json | jq -r '
- .matches[]
- | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
- | .vulnerability.id' | wc -l)
-
- if [ "$TOTAL_HIGH_SEVERITY" -gt 0 ]; then
- echo "Total vulnerabilities found with CVSS >= 4.0: $TOTAL_HIGH_SEVERITY"
- echo ""
- echo "All high-severity vulnerabilities (including allowlisted):"
- grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
- ["Package", "Version", "Vulnerability ID", "CVSS Score", "Allowlisted"],
- (.matches[]
- | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
- | [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, (if (.vulnerability.id as $id | $allow | index($id)) then "YES" else "NO" end)])
- | @tsv' | column -t -s $'\t'
- echo ""
- fi
-
- HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
- .matches[]
- | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
- | select((.vulnerability.id as $id | $allow | index($id) | not))
- | .vulnerability.id' | wc -l)
-
- if [ "$HIGH_SEVERITY_COUNT" -gt 0 ]; then
- echo ""
- echo "=========================================="
- echo "ERROR: Security Scan Failed"
- echo "=========================================="
- echo "Found $HIGH_SEVERITY_COUNT non-allowlisted vulnerabilities with CVSS score >= 4.0 in litellm:latest"
- echo ""
- echo "These vulnerabilities are NOT in the allowlist and must be addressed."
- echo "Current allowlisted CVEs: ${ALLOWED_CVES[*]}"
- echo ""
- echo "Detailed vulnerability report:"
- echo ""
- grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
- ["Package", "Version", "Vulnerability ID", "CVSS Score", "Severity", "Fix Version", "Description"],
- (.matches[]
- | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
- | select((.vulnerability.id as $id | $allow | index($id) | not))
- | [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description])
- | @tsv' | column -t -s $'\t'
- echo ""
- echo "=========================================="
- echo "Action Required:"
- echo "=========================================="
- echo "1. If a fix is available, update the package to the fixed version"
- echo "2. If the vulnerability is not applicable or has no fix:"
- echo " - Add the CVE/GHSA ID to ALLOWED_CVES array in ci_cd/security_scans.sh"
- echo " - Add a comment explaining why it's safe to ignore"
- echo ""
- echo "Note: Some vulnerabilities may have multiple IDs (CVE-XXXX and GHSA-XXXX)."
- echo "Add all relevant IDs to the allowlist if they refer to the same issue."
- echo "=========================================="
- echo ""
- exit 1
- else
- echo "No high-severity vulnerabilities (CVSS >= 4.0) found in litellm:latest"
- fi
-
- echo "Grype scans completed successfully"
-}
-
-# Main execution
-main() {
- echo "Installing security scanning tools..."
- install_trivy
- install_grype
-
- # echo "Running secret detection scans..."
- # run_secret_detection
-
- echo "Running filesystem vulnerability scans..."
- run_trivy_scans
-
- echo "Running Docker image vulnerability scans..."
- run_grype_scans
-
- echo "All security scans completed successfully!"
-}
-
-# Execute main function
-main "$@"
diff --git a/docker/build_admin_ui.sh b/docker/build_admin_ui.sh
index 5373ad0e3d9..efb2bac3535 100755
--- a/docker/build_admin_ui.sh
+++ b/docker/build_admin_ui.sh
@@ -40,11 +40,22 @@ else
exit 1
fi
fi
-curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash
+NVM_VERSION="v0.40.4"
+NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f"
+NVM_SCRIPT=$(mktemp)
+trap 'rm -f "$NVM_SCRIPT"' EXIT
+curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT"
+if command -v sha256sum &>/dev/null; then
+ echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | sha256sum -c -
+elif command -v shasum &>/dev/null; then
+ echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | shasum -a 256 -c -
+else
+ echo "No sha256 tool found; cannot verify nvm checksum"; exit 1
+fi || { echo "nvm checksum verification failed"; exit 1; }
+bash "$NVM_SCRIPT"
source ~/.nvm/nvm.sh
nvm install v18.17.0
nvm use v18.17.0
-npm install -g npm
# copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json
cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json
diff --git a/docs/my-website/.trivyignore b/docs/my-website/.trivyignore
deleted file mode 100644
index 977504f2670..00000000000
--- a/docs/my-website/.trivyignore
+++ /dev/null
@@ -1,7 +0,0 @@
-# js-yaml CVE-2025-64718
-# This vulnerability is not applicable because we've forced js-yaml to version 4.1.1
-# via npm overrides in package.json. Trivy incorrectly reports this based on
-# dependency requirements in the lockfile, but the actual installed version is 4.1.1.
-# Verified with: npm list js-yaml
-CVE-2025-64718
-
diff --git a/scripts/install.sh b/scripts/install.sh
new file mode 100644
index 00000000000..03ae31cd18b
--- /dev/null
+++ b/scripts/install.sh
@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+# LiteLLM Installer
+# Usage: curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/install.sh | sh
+#
+# NOTE: set -e without pipefail for POSIX sh compatibility (dash on Ubuntu/Debian
+# ignores the shebang when invoked as `sh` and does not support `pipefail`).
+set -eu
+
+MIN_PYTHON_MAJOR=3
+MIN_PYTHON_MINOR=9
+
+# NOTE: before merging, this must stay as "litellm[proxy]" to install from PyPI.
+LITELLM_PACKAGE="litellm[proxy]"
+
+# ── colours ────────────────────────────────────────────────────────────────
+if [ -t 1 ]; then
+ BOLD='\033[1m'
+ GREEN='\033[38;2;78;186;101m'
+ GREY='\033[38;2;153;153;153m'
+ RESET='\033[0m'
+else
+ BOLD='' GREEN='' GREY='' RESET=''
+fi
+
+info() { printf "${GREY} %s${RESET}\n" "$*"; }
+success() { printf "${GREEN} ✔ %s${RESET}\n" "$*"; }
+header() { printf "${BOLD} %s${RESET}\n" "$*"; }
+die() { printf "\n Error: %s\n\n" "$*" >&2; exit 1; }
+
+# ── banner ─────────────────────────────────────────────────────────────────
+echo ""
+cat << 'EOF'
+ ██╗ ██╗████████╗███████╗██╗ ██╗ ███╗ ███╗
+ ██║ ██║╚══██╔══╝██╔════╝██║ ██║ ████╗ ████║
+ ██║ ██║ ██║ █████╗ ██║ ██║ ██╔████╔██║
+ ██║ ██║ ██║ ██╔══╝ ██║ ██║ ██║╚██╔╝██║
+ ███████╗██║ ██║ ███████╗███████╗███████╗██║ ╚═╝ ██║
+ ╚══════╝╚═╝ ╚═╝ ╚══════╝╚══════╝╚══════╝╚═╝ ╚═╝
+EOF
+printf " ${BOLD}LiteLLM Installer${RESET} ${GREY}— unified gateway for 100+ LLM providers${RESET}\n\n"
+
+# ── OS detection ───────────────────────────────────────────────────────────
+OS="$(uname -s)"
+ARCH="$(uname -m)"
+
+case "$OS" in
+ Darwin) PLATFORM="macOS ($ARCH)" ;;
+ Linux) PLATFORM="Linux ($ARCH)" ;;
+ *) die "Unsupported OS: $OS. LiteLLM supports macOS and Linux." ;;
+esac
+
+info "Platform: $PLATFORM"
+
+# ── Python detection ───────────────────────────────────────────────────────
+PYTHON_BIN=""
+for candidate in python3 python; do
+ if command -v "$candidate" >/dev/null 2>&1; then
+ major="$("$candidate" -c 'import sys; print(sys.version_info.major)' 2>/dev/null || true)"
+ minor="$("$candidate" -c 'import sys; print(sys.version_info.minor)' 2>/dev/null || true)"
+ if [ "${major:-0}" -ge "$MIN_PYTHON_MAJOR" ] && [ "${minor:-0}" -ge "$MIN_PYTHON_MINOR" ]; then
+ PYTHON_BIN="$(command -v "$candidate")"
+ info "Python: $("$candidate" --version 2>&1)"
+ break
+ fi
+ fi
+done
+
+if [ -z "$PYTHON_BIN" ]; then
+ die "Python ${MIN_PYTHON_MAJOR}.${MIN_PYTHON_MINOR}+ is required but not found.
+ Install it from https://python.org/downloads or via your package manager:
+ macOS: brew install python@3
+ Ubuntu: sudo apt install python3 python3-pip"
+fi
+
+# ── pip detection ──────────────────────────────────────────────────────────
+if ! "$PYTHON_BIN" -m pip --version >/dev/null 2>&1; then
+ die "pip is not available. Install it with:
+ $PYTHON_BIN -m ensurepip --upgrade"
+fi
+
+# ── install ────────────────────────────────────────────────────────────────
+echo ""
+header "Installing litellm[proxy]…"
+echo ""
+
+"$PYTHON_BIN" -m pip install --only-binary :all: --upgrade "${LITELLM_PACKAGE}" \
+ || die "pip install failed. Try manually: $PYTHON_BIN -m pip install --only-binary :all: '${LITELLM_PACKAGE}'"
+
+# ── find the litellm binary installed by pip for this Python ───────────────
+# sysconfig.get_path('scripts') is where pip puts console scripts — reliable
+# even when the Python lives in a libexec/ symlink tree (e.g. Homebrew).
+SCRIPTS_DIR="$("$PYTHON_BIN" -c 'import sysconfig; print(sysconfig.get_path("scripts"))')"
+LITELLM_BIN="${SCRIPTS_DIR}/litellm"
+
+if [ ! -x "$LITELLM_BIN" ]; then
+ # Fall back to user-base bin (pip install --user)
+ USER_BIN="$("$PYTHON_BIN" -c 'import site; print(site.getuserbase())')/bin"
+ LITELLM_BIN="${USER_BIN}/litellm"
+fi
+
+if [ ! -x "$LITELLM_BIN" ]; then
+ die "litellm binary not found after install. Try: $PYTHON_BIN -m pip install --user '${LITELLM_PACKAGE}'"
+fi
+
+# ── success banner ─────────────────────────────────────────────────────────
+echo ""
+success "LiteLLM installed"
+
+installed_ver="$("$LITELLM_BIN" --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)"
+[ -n "$installed_ver" ] && info "Version: $installed_ver"
+
+# ── PATH hint ──────────────────────────────────────────────────────────────
+if ! command -v litellm >/dev/null 2>&1; then
+ info "Note: add litellm to your PATH: export PATH=\"\$PATH:${SCRIPTS_DIR}\""
+fi
+
+# ── launch setup wizard ────────────────────────────────────────────────────
+echo ""
+printf " ${BOLD}Run the interactive setup wizard?${RESET} ${GREY}(Y/n)${RESET}: "
+# /dev/tty may be unavailable in Docker/CI — default to yes if it can't be read
+answer=""
+if [ -r /dev/tty ]; then
+ read -r answer /dev/null; then
- # Install nvm
- curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash
+ # Install nvm with checksum verification
+ NVM_VERSION="v0.40.4"
+ NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f"
+ NVM_SCRIPT=$(mktemp)
+ trap 'rm -f "$NVM_SCRIPT"' EXIT
+ curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT"
+ if command -v sha256sum &>/dev/null; then
+ echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | sha256sum -c -
+ elif command -v shasum &>/dev/null; then
+ echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | shasum -a 256 -c -
+ else
+ echo "No sha256 tool found; cannot verify nvm checksum"; exit 1
+ fi || { echo "nvm checksum verification failed"; exit 1; }
+ bash "$NVM_SCRIPT"
# Source nvm script in the current session
export NVM_DIR="$HOME/.nvm"
diff --git a/ui/litellm-dashboard/build_ui_custom_path.sh b/ui/litellm-dashboard/build_ui_custom_path.sh
index f947f87d3b7..a92927f8ea7 100755
--- a/ui/litellm-dashboard/build_ui_custom_path.sh
+++ b/ui/litellm-dashboard/build_ui_custom_path.sh
@@ -12,8 +12,20 @@ UI_BASE_PATH="$1"
# Check if nvm is not installed
if ! command -v nvm &> /dev/null; then
- # Install nvm
- curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash
+ # Install nvm with checksum verification
+ NVM_VERSION="v0.40.4"
+ NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f"
+ NVM_SCRIPT=$(mktemp)
+ trap 'rm -f "$NVM_SCRIPT"' EXIT
+ curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT"
+ if command -v sha256sum &>/dev/null; then
+ echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | sha256sum -c -
+ elif command -v shasum &>/dev/null; then
+ echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | shasum -a 256 -c -
+ else
+ echo "No sha256 tool found; cannot verify nvm checksum"; exit 1
+ fi || { echo "nvm checksum verification failed"; exit 1; }
+ bash "$NVM_SCRIPT"
# Source nvm script in the current session
export NVM_DIR="$HOME/.nvm"