diff --git a/.circleci/config.yml b/.circleci/config.yml index 333047ab57c..d99c485af94 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,3404 +112,3404 @@ jobs: python -m mypy . cd .. no_output_timeout: 10m - # local_testing_part1: - # docker: - # - image: cimg/python:3.12 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - # parallelism: 4 - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Show git commit hash - # command: | - # echo "Git commit hash: $CIRCLE_SHA1" - - # - restore_cache: - # keys: - # - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r .circleci/requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-asyncio==0.21.1" - # pip install "pytest-cov==5.0.0" - # pip install "mypy==1.18.2" - # pip install "google-generativeai==0.3.2" - # pip install "google-cloud-aiplatform==1.43.0" - # pip install pyarrow - # pip install "boto3==1.36.0" - # pip install "aioboto3==13.4.0" - # pip install langchain - # pip install lunary==0.2.5 - # pip install "azure-identity==1.16.1" - # pip install "langfuse==2.59.7" - # pip install "logfire==0.29.0" - # pip install numpydoc - # pip install traceloop-sdk==0.21.1 - # pip install opentelemetry-api==1.25.0 - # pip install opentelemetry-sdk==1.25.0 - # pip install opentelemetry-exporter-otlp==1.25.0 - # pip install openai==1.100.1 - # pip install prisma==0.11.0 - # pip install "detect_secrets==1.5.0" - # pip install "httpx==0.24.1" - # pip install "respx==0.22.0" - # pip install fastapi - # pip install "gunicorn==21.2.0" - # pip install "anyio==4.2.0" - # pip install "aiodynamo==23.10.1" - # pip install "asyncio==3.4.3" - # pip install "apscheduler==3.10.4" - # pip install "PyGithub==1.59.1" - # pip install argon2-cffi - # pip install "pytest-mock==3.12.0" - # pip install python-multipart - # pip install google-cloud-aiplatform - # pip install prometheus-client==0.20.0 - # pip install "pydantic==2.10.2" - # pip install "diskcache==5.6.1" - # pip install "Pillow==10.3.0" - # pip install "jsonschema==4.22.0" - # pip install "pytest-xdist==3.6.1" - # pip install "pytest-timeout==2.2.0" - # pip install "websockets==13.1.0" - # pip install semantic_router --no-deps - # pip install aurelio_sdk --no-deps - # pip uninstall posthog -y - # - setup_litellm_enterprise_pip - # - save_cache: - # paths: - # - ./venv - # key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - # - run: - # name: Run prisma ./docker/entrypoint.sh - # command: | - # set +e - # chmod +x docker/entrypoint.sh - # ./docker/entrypoint.sh - # set -e - # - run: - # name: Black Formatting - # command: | - # cd litellm - # python -m pip install black - # python -m black . - # cd .. - - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests (Part 1 - A-M) - # command: | - # mkdir test-results - - # # Discover test files (A-M) - # TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py") - - # echo "$TEST_FILES" | circleci tests run \ - # --split-by=timings \ - # --verbose \ - # --command="xargs python -m pytest \ - # -vv \ - # --cov=litellm \ - # --cov-report=xml \ - # --junitxml=test-results/junit.xml \ - # --durations=20 \ - # -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ - # -n 4 \ - # --timeout=300 \ - # --timeout_method=thread" - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml local_testing_part1_coverage.xml - # mv .coverage local_testing_part1_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - local_testing_part1_coverage.xml - # - local_testing_part1_coverage - # local_testing_part2: - # docker: - # - image: cimg/python:3.12 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - # parallelism: 4 - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Show git commit hash - # command: | - # echo "Git commit hash: $CIRCLE_SHA1" - - # - restore_cache: - # keys: - # - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r .circleci/requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-asyncio==0.21.1" - # pip install "pytest-cov==5.0.0" - # pip install "mypy==1.18.2" - # pip install "google-generativeai==0.3.2" - # pip install "google-cloud-aiplatform==1.43.0" - # pip install pyarrow - # pip install "boto3==1.36.0" - # pip install "aioboto3==13.4.0" - # pip install langchain - # pip install lunary==0.2.5 - # pip install "azure-identity==1.16.1" - # pip install "langfuse==2.59.7" - # pip install "logfire==0.29.0" - # pip install numpydoc - # pip install traceloop-sdk==0.21.1 - # pip install opentelemetry-api==1.25.0 - # pip install opentelemetry-sdk==1.25.0 - # pip install opentelemetry-exporter-otlp==1.25.0 - # pip install openai==1.100.1 - # pip install prisma==0.11.0 - # pip install "detect_secrets==1.5.0" - # pip install "httpx==0.24.1" - # pip install "respx==0.22.0" - # pip install fastapi - # pip install "gunicorn==21.2.0" - # pip install "anyio==4.2.0" - # pip install "aiodynamo==23.10.1" - # pip install "asyncio==3.4.3" - # pip install "apscheduler==3.10.4" - # pip install "PyGithub==1.59.1" - # pip install argon2-cffi - # pip install "pytest-mock==3.12.0" - # pip install python-multipart - # pip install google-cloud-aiplatform - # pip install prometheus-client==0.20.0 - # pip install "pydantic==2.10.2" - # pip install "diskcache==5.6.1" - # pip install "Pillow==10.3.0" - # pip install "jsonschema==4.22.0" - # pip install "pytest-xdist==3.6.1" - # pip install "pytest-timeout==2.2.0" - # pip install "websockets==13.1.0" - # pip install semantic_router --no-deps - # pip install aurelio_sdk --no-deps - # pip uninstall posthog -y - # - setup_litellm_enterprise_pip - # - save_cache: - # paths: - # - ./venv - # key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - # - run: - # name: Run prisma ./docker/entrypoint.sh - # command: | - # set +e - # chmod +x docker/entrypoint.sh - # ./docker/entrypoint.sh - # set -e - # - run: - # name: Black Formatting - # command: | - # cd litellm - # python -m pip install black - # python -m black . - # cd .. - - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests (Part 2 - N-Z) - # command: | - # mkdir test-results - - # # Discover test files (N-Z) - # TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py") - - # echo "$TEST_FILES" | circleci tests run \ - # --split-by=timings \ - # --verbose \ - # --command="xargs python -m pytest \ - # -vv \ - # --cov=litellm \ - # --cov-report=xml \ - # --junitxml=test-results/junit.xml \ - # --durations=20 \ - # -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ - # -n 4 \ - # --timeout=300 \ - # --timeout_method=thread" - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml local_testing_part2_coverage.xml - # mv .coverage local_testing_part2_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - local_testing_part2_coverage.xml - # - local_testing_part2_coverage - # langfuse_logging_unit_tests: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Show git commit hash - # command: | - # echo "Git commit hash: $CIRCLE_SHA1" - - # - restore_cache: - # keys: - # - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r .circleci/requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-asyncio==0.21.1" - # pip install "pytest-cov==5.0.0" - # pip install "mypy==1.18.2" - # pip install "google-generativeai==0.3.2" - # pip install "google-cloud-aiplatform==1.43.0" - # pip install pyarrow - # pip install "boto3==1.36.0" - # pip install "aioboto3==13.4.0" - # pip install langchain - # pip install lunary==0.2.5 - # pip install "azure-identity==1.16.1" - # pip install "langfuse==2.59.7" - # pip install "logfire==0.29.0" - # pip install numpydoc - # pip install traceloop-sdk==0.21.1 - # pip install opentelemetry-api==1.25.0 - # pip install opentelemetry-sdk==1.25.0 - # pip install opentelemetry-exporter-otlp==1.25.0 - # pip install openai==1.100.1 - # pip install prisma==0.11.0 - # pip install "detect_secrets==1.5.0" - # pip install "httpx==0.24.1" - # pip install "respx==0.22.0" - # pip install fastapi - # pip install "gunicorn==21.2.0" - # pip install "anyio==4.2.0" - # pip install "aiodynamo==23.10.1" - # pip install "asyncio==3.4.3" - # pip install "apscheduler==3.10.4" - # pip install "PyGithub==1.59.1" - # pip install argon2-cffi - # pip install "pytest-mock==3.12.0" - # pip install python-multipart - # pip install google-cloud-aiplatform - # pip install prometheus-client==0.20.0 - # pip install "pydantic==2.10.2" - # pip install "diskcache==5.6.1" - # pip install "Pillow==10.3.0" - # pip install "jsonschema==4.22.0" - # pip install "websockets==13.1.0" - # - setup_litellm_enterprise_pip - # - save_cache: - # paths: - # - ./venv - # key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - # - run: - # name: Run prisma ./docker/entrypoint.sh - # command: | - # set +e - # chmod +x docker/entrypoint.sh - # ./docker/entrypoint.sh - # set -e - - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse" - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml langfuse_coverage.xml - # mv .coverage langfuse_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - langfuse_coverage.xml - # - langfuse_coverage - # caching_unit_tests: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: DNS lookup for Redis host - # command: | - # sudo apt-get update - # sudo apt-get install -y dnsutils - # dig redis-19899.c239.us-east-1-2.ec2.redns.redis-cloud.com +short - # - run: - # name: Show git commit hash - # command: | - # echo "Git commit hash: $CIRCLE_SHA1" - - # - restore_cache: - # keys: - # - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r .circleci/requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-asyncio==0.21.1" - # pip install "pytest-cov==5.0.0" - # pip install "mypy==1.18.2" - # pip install "google-generativeai==0.3.2" - # pip install "google-cloud-aiplatform==1.43.0" - # pip install pyarrow - # pip install "boto3==1.36.0" - # pip install "aioboto3==13.4.0" - # pip install langchain - # pip install lunary==0.2.5 - # pip install "azure-identity==1.16.1" - # pip install "langfuse==2.59.7" - # pip install "logfire==0.29.0" - # pip install numpydoc - # pip install traceloop-sdk==0.21.1 - # pip install opentelemetry-api==1.25.0 - # pip install opentelemetry-sdk==1.25.0 - # pip install opentelemetry-exporter-otlp==1.25.0 - # pip install openai==1.100.1 - # pip install prisma==0.11.0 - # pip install "detect_secrets==1.5.0" - # pip install "httpx==0.24.1" - # pip install "respx==0.22.0" - # pip install fastapi - # pip install "gunicorn==21.2.0" - # pip install "anyio==4.2.0" - # pip install "aiodynamo==23.10.1" - # pip install "asyncio==3.4.3" - # pip install "apscheduler==3.10.4" - # pip install "PyGithub==1.59.1" - # pip install argon2-cffi - # pip install "pytest-mock==3.12.0" - # pip install python-multipart - # pip install google-cloud-aiplatform - # pip install prometheus-client==0.20.0 - # pip install "pydantic==2.10.2" - # pip install "diskcache==5.6.1" - # pip install "Pillow==10.3.0" - # pip install "jsonschema==4.22.0" - # pip install "websockets==13.1.0" - # - setup_litellm_enterprise_pip - # - save_cache: - # paths: - # - ./venv - # key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - # - run: - # name: Run prisma ./docker/entrypoint.sh - # command: | - # set +e - # chmod +x docker/entrypoint.sh - # ./docker/entrypoint.sh - # set -e - - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "caching or cache" - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml caching_coverage.xml - # mv .coverage caching_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - caching_coverage.xml - # - caching_coverage - # auth_ui_unit_tests: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-asyncio==0.21.1" - # pip install "pytest-cov==5.0.0" - # - save_cache: - # paths: - # - ./venv - # key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - # - run: - # name: Run prisma ./docker/entrypoint.sh - # command: | - # set +e - # chmod +x docker/entrypoint.sh - # ./docker/entrypoint.sh - # set -e - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/proxy_admin_ui_tests -x --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml auth_ui_unit_tests_coverage.xml - # mv .coverage auth_ui_unit_tests_coverage - - # # Store test results - # - store_test_results: - # path: test-results - - # - persist_to_workspace: - # root: . - # paths: - # - auth_ui_unit_tests_coverage.xml - # - auth_ui_unit_tests_coverage - - # litellm_router_testing: # Runs all tests with the "router" keyword - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "respx==0.22.0" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-asyncio==0.21.1" - # pip install "pytest-xdist==3.6.1" - # pip install semantic_router --no-deps - # pip install aurelio_sdk --no-deps - # # Run pytest and generate JUnit XML report - # - setup_litellm_enterprise_pip - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest tests/local_testing --cov=litellm --cov-report=xml -vv -k "router" -v --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml litellm_router_coverage.xml - # mv .coverage litellm_router_coverage - # # Store test results - # - store_test_results: - # path: test-results - - # - persist_to_workspace: - # root: . - # paths: - # - litellm_router_coverage.xml - # - litellm_router_coverage - - # litellm_router_unit_testing: # Runs all tests with the "router" keyword - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "respx==0.22.0" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-asyncio==0.21.1" - # pip install semantic_router --no-deps - # pip install aurelio_sdk --no-deps - # pip install "pytest-xdist==3.6.1" - # # Run pytest and generate JUnit XML report - # - setup_litellm_enterprise_pip - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/router_unit_tests --cov=litellm --cov-report=xml -x -s --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml litellm_router_unit_coverage.xml - # mv .coverage litellm_router_unit_coverage - # # Store test results - # - store_test_results: - # path: test-results - - # - persist_to_workspace: - # root: . - # paths: - # - litellm_router_unit_coverage.xml - # - litellm_router_unit_coverage - # litellm_security_tests: - # machine: - # image: ubuntu-2204:2023.10.1 - # resource_class: xlarge - # working_directory: ~/project - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Show git commit hash - # command: | - # echo "Git commit hash: $CIRCLE_SHA1" - # - run: - # name: Install Docker CLI (In case it's not already installed) - # command: | - # curl -fsSL https://get.docker.com | sh - # sudo usermod -aG docker $USER - # docker version - # - run: - # name: Install Python 3.13 - # command: | - # curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - # bash miniconda.sh -b -p $HOME/miniconda - # export PATH="$HOME/miniconda/bin:$PATH" - # conda init bash - # source ~/.bashrc - # conda create -n myenv python=3.13 -y - # conda activate myenv - # python --version - # - run: - # name: Install Dependencies - # command: | - # export PATH="$HOME/miniconda/bin:$PATH" - # source $HOME/miniconda/etc/profile.d/conda.sh - # conda activate myenv - # python --version - # which python - # pip install --upgrade typing-extensions>=4.12.0 - # pip install "pytest==7.3.1" - # pip install "pytest-asyncio==0.21.1" - # pip install aiohttp - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-mock==3.12.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "mypy==1.18.2" - # pip install "google-generativeai==0.3.2" - # pip install "google-cloud-aiplatform==1.43.0" - # pip install pyarrow - # pip install "boto3==1.36.0" - # pip install "aioboto3==13.4.0" - # pip install langchain - # pip install "langfuse>=2.0.0" - # pip install "logfire==0.29.0" - # pip install numpydoc - # pip install prisma - # pip install fastapi - # pip install jsonschema - # pip install "httpx==0.24.1" - # pip install "gunicorn==21.2.0" - # pip install "anyio==3.7.1" - # pip install "aiodynamo==23.10.1" - # pip install "asyncio==3.4.3" - # pip install "PyGithub==1.59.1" - # pip install "openai==1.100.1" - # pip install "pytest-cov==5.0.0" - # pip install "apscheduler" - # - run: - # name: Install dockerize - # command: | - # wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - # sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - # rm dockerize-linux-amd64-v0.6.1.tar.gz - # - run: - # name: Start PostgreSQL Database - # command: | - # docker run -d \ - # --name postgres-db \ - # -e POSTGRES_USER=postgres \ - # -e POSTGRES_PASSWORD=postgres \ - # -e POSTGRES_DB=circle_test \ - # -p 5432:5432 \ - # postgres:14 - # - run: - # name: Wait for PostgreSQL to be ready - # command: dockerize -wait tcp://localhost:5432 -timeout 1m - # - run: - # name: Set DATABASE_URL environment variable - # command: | - # echo 'export DATABASE_URL="postgresql://postgres:postgres@localhost:5432/circle_test"' >> $BASH_ENV - # source $BASH_ENV - # - run: - # name: Run Security Scans - # command: | - # chmod +x ci_cd/security_scans.sh - # ./ci_cd/security_scans.sh - # - run: - # name: Run prisma ./docker/entrypoint.sh - # command: | - # export PATH="$HOME/miniconda/bin:$PATH" - # source $HOME/miniconda/etc/profile.d/conda.sh - # conda activate myenv - # set +e - # chmod +x docker/entrypoint.sh - # ./docker/entrypoint.sh - # set -e - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # export PATH="$HOME/miniconda/bin:$PATH" - # source $HOME/miniconda/etc/profile.d/conda.sh - # conda activate myenv - # pwd - # ls - # python -m pytest tests/proxy_security_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml litellm_security_tests_coverage.xml - # mv .coverage litellm_security_tests_coverage - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - litellm_security_tests_coverage.xml - # - litellm_security_tests_coverage - # # Split proxy unit tests into 3 jobs for faster execution and better debugging - # # test_key_generate_prisma runs separately without parallel execution to avoid event loop issues with logging worker - # litellm_proxy_unit_testing_key_generation: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - # resource_class: large - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Show git commit hash - # command: | - # echo "Git commit hash: $CIRCLE_SHA1" - # - run: - # name: Install PostgreSQL - # command: | - # sudo apt-get update - # sudo apt-get install -y postgresql-14 postgresql-contrib-14 - # - restore_cache: - # keys: - # - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r .circleci/requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-asyncio==0.21.1" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-timeout==2.2.0" - # pip install "pytest-forked==1.6.0" - # pip install "mypy==1.18.2" - # pip install "google-generativeai==0.3.2" - # pip install "google-cloud-aiplatform==1.43.0" - # pip install "google-genai==1.22.0" - # pip install pyarrow - # pip install "boto3==1.36.0" - # pip install "aioboto3==13.4.0" - # pip install langchain - # pip install lunary==0.2.5 - # pip install "azure-identity==1.16.1" - # pip install "langfuse==2.59.7" - # pip install "logfire==0.29.0" - # pip install numpydoc - # pip install traceloop-sdk==0.21.1 - # pip install opentelemetry-api==1.25.0 - # pip install opentelemetry-sdk==1.25.0 - # pip install opentelemetry-exporter-otlp==1.25.0 - # pip install openai==1.100.1 - # pip install prisma==0.11.0 - # pip install "detect_secrets==1.5.0" - # pip install "httpx==0.24.1" - # pip install "respx==0.22.0" - # pip install fastapi - # pip install "gunicorn==21.2.0" - # pip install "anyio==4.2.0" - # pip install "aiodynamo==23.10.1" - # pip install "asyncio==3.4.3" - # pip install "apscheduler==3.10.4" - # pip install "PyGithub==1.59.1" - # pip install argon2-cffi - # pip install "pytest-mock==3.12.0" - # pip install python-multipart - # pip install google-cloud-aiplatform - # pip install prometheus-client==0.20.0 - # pip install "pydantic==2.10.2" - # pip install "diskcache==5.6.1" - # pip install "Pillow==10.3.0" - # pip install "jsonschema==4.22.0" - # pip install "pytest-postgresql==7.0.1" - # pip install "fakeredis==2.28.1" - # - setup_litellm_enterprise_pip - # - save_cache: - # paths: - # - ./venv - # key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - # - run: - # name: Run prisma ./docker/entrypoint.sh - # command: | - # set +e - # chmod +x docker/entrypoint.sh - # ./docker/entrypoint.sh - # set -e - # - run: - # name: Run key generation tests (no parallel execution to avoid event loop issues) - # command: | - # pwd - # ls - # # Run without -n flag to avoid pytest-xdist event loop conflicts with logging worker - # python -m pytest tests/proxy_unit_tests/test_key_generate_prisma.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-key-generation.xml --durations=10 --timeout=300 -vv --log-cli-level=INFO - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml litellm_proxy_unit_tests_key_generation_coverage.xml - # mv .coverage litellm_proxy_unit_tests_key_generation_coverage - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - litellm_proxy_unit_tests_key_generation_coverage.xml - # - litellm_proxy_unit_tests_key_generation_coverage - # litellm_proxy_unit_testing_part1: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - # resource_class: large - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Show git commit hash - # command: | - # echo "Git commit hash: $CIRCLE_SHA1" - # - run: - # name: Install PostgreSQL - # command: | - # sudo apt-get update - # sudo apt-get install -y postgresql-14 postgresql-contrib-14 - # - restore_cache: - # keys: - # - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r .circleci/requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-asyncio==0.21.1" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-timeout==2.2.0" - # pip install "pytest-forked==1.6.0" - # pip install "mypy==1.18.2" - # pip install "google-generativeai==0.3.2" - # pip install "google-cloud-aiplatform==1.43.0" - # pip install "google-genai==1.22.0" - # pip install pyarrow - # pip install "boto3==1.36.0" - # pip install "aioboto3==13.4.0" - # pip install langchain - # pip install lunary==0.2.5 - # pip install "azure-identity==1.16.1" - # pip install "langfuse==2.59.7" - # pip install "logfire==0.29.0" - # pip install numpydoc - # pip install traceloop-sdk==0.21.1 - # pip install opentelemetry-api==1.25.0 - # pip install opentelemetry-sdk==1.25.0 - # pip install opentelemetry-exporter-otlp==1.25.0 - # pip install openai==1.100.1 - # pip install prisma==0.11.0 - # pip install "detect_secrets==1.5.0" - # pip install "httpx==0.24.1" - # pip install "respx==0.22.0" - # pip install fastapi - # pip install "gunicorn==21.2.0" - # pip install "anyio==4.2.0" - # pip install "aiodynamo==23.10.1" - # pip install "asyncio==3.4.3" - # pip install "apscheduler==3.10.4" - # pip install "PyGithub==1.59.1" - # pip install argon2-cffi - # pip install "pytest-mock==3.12.0" - # pip install python-multipart - # pip install google-cloud-aiplatform - # pip install prometheus-client==0.20.0 - # pip install "pydantic==2.10.2" - # pip install "diskcache==5.6.1" - # pip install "Pillow==10.3.0" - # pip install "jsonschema==4.22.0" - # pip install "pytest-postgresql==7.0.1" - # pip install "fakeredis==2.28.1" - # pip install "pytest-xdist==3.6.1" - # - setup_litellm_enterprise_pip - # - save_cache: - # paths: - # - ./venv - # key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - # - run: - # name: Run prisma ./docker/entrypoint.sh - # command: | - # set +e - # chmod +x docker/entrypoint.sh - # ./docker/entrypoint.sh - # set -e - # - run: - # name: Run proxy unit tests (part 1 - auth checks only, key generation in separate job) - # command: | - # pwd - # ls - # # Run auth tests with parallel execution (test_key_generate_prisma moved to separate job to avoid event loop issues) - # python -m pytest tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part1.xml --durations=10 -n 8 --timeout=300 -vv --log-cli-level=INFO - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml litellm_proxy_unit_tests_part1_coverage.xml - # mv .coverage litellm_proxy_unit_tests_part1_coverage - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - litellm_proxy_unit_tests_part1_coverage.xml - # - litellm_proxy_unit_tests_part1_coverage - # litellm_proxy_unit_testing_part2: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - # resource_class: large - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Show git commit hash - # command: | - # echo "Git commit hash: $CIRCLE_SHA1" - # - run: - # name: Install PostgreSQL - # command: | - # sudo apt-get update - # sudo apt-get install -y postgresql-14 postgresql-contrib-14 - # - restore_cache: - # keys: - # - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r .circleci/requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-asyncio==0.21.1" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-timeout==2.2.0" - # pip install "pytest-forked==1.6.0" - # pip install "mypy==1.18.2" - # pip install "google-generativeai==0.3.2" - # pip install "google-cloud-aiplatform==1.43.0" - # pip install "google-genai==1.22.0" - # pip install pyarrow - # pip install "boto3==1.36.0" - # pip install "aioboto3==13.4.0" - # pip install langchain - # pip install lunary==0.2.5 - # pip install "azure-identity==1.16.1" - # pip install "langfuse==2.59.7" - # pip install "logfire==0.29.0" - # pip install numpydoc - # pip install traceloop-sdk==0.21.1 - # pip install opentelemetry-api==1.25.0 - # pip install opentelemetry-sdk==1.25.0 - # pip install opentelemetry-exporter-otlp==1.25.0 - # pip install openai==1.100.1 - # pip install prisma==0.11.0 - # pip install "detect_secrets==1.5.0" - # pip install "httpx==0.24.1" - # pip install "respx==0.22.0" - # pip install fastapi - # pip install "gunicorn==21.2.0" - # pip install "anyio==4.2.0" - # pip install "aiodynamo==23.10.1" - # pip install "asyncio==3.4.3" - # pip install "apscheduler==3.10.4" - # pip install "PyGithub==1.59.1" - # pip install argon2-cffi - # pip install "pytest-mock==3.12.0" - # pip install python-multipart - # pip install google-cloud-aiplatform - # pip install prometheus-client==0.20.0 - # pip install "pydantic==2.10.2" - # pip install "diskcache==5.6.1" - # pip install "Pillow==10.3.0" - # pip install "jsonschema==4.22.0" - # pip install "pytest-postgresql==7.0.1" - # pip install "fakeredis==2.28.1" - # pip install "pytest-xdist==3.6.1" - # - setup_litellm_enterprise_pip - # - save_cache: - # paths: - # - ./venv - # key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - # - run: - # name: Run prisma ./docker/entrypoint.sh - # command: | - # set +e - # chmod +x docker/entrypoint.sh - # ./docker/entrypoint.sh - # set -e - # - run: - # name: Run proxy unit tests (part 2 - remaining tests) - # command: | - # pwd - # ls - # python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part2.xml --durations=10 -n 8 --timeout=300 -vv --log-cli-level=INFO - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml litellm_proxy_unit_tests_part2_coverage.xml - # mv .coverage litellm_proxy_unit_tests_part2_coverage - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - litellm_proxy_unit_tests_part2_coverage.xml - # - litellm_proxy_unit_tests_part2_coverage - # litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword - # docker: - # - image: cimg/python:3.13.1 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # pip install wheel - # pip install --upgrade pip wheel setuptools - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "respx==0.22.0" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-asyncio==0.21.1" - # pip install "pytest-cov==5.0.0" - # # Run pytest and generate JUnit XML report - # - setup_litellm_enterprise_pip - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest tests/local_testing/ -vv -k "assistants" --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml litellm_assistants_api_coverage.xml - # mv .coverage litellm_assistants_api_coverage - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - litellm_assistants_api_coverage.xml - # - litellm_assistants_api_coverage - # llm_translation_testing: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "respx==0.22.0" - # pip install "pytest-xdist==3.6.1" - # pip install "pytest-timeout==2.2.0" - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # pwd - # ls - # # Add --timeout to kill hanging tests after 120s (2 min) - # # Add --durations=20 to show 20 slowest tests for debugging - # python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml llm_translation_coverage.xml - # mv .coverage llm_translation_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - llm_translation_coverage.xml - # - llm_translation_coverage - # mcp_testing: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "respx==0.22.0" - # pip install "pydantic==2.11.0" - # pip install "mcp==1.25.0" - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/mcp_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml mcp_coverage.xml - # mv .coverage mcp_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - mcp_coverage.xml - # - mcp_coverage - # guardrails_testing: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "respx==0.22.0" - # pip install "pydantic==2.10.2" - # pip install "boto3==1.36.0" - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/guardrails_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml guardrails_coverage.xml - # mv .coverage guardrails_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - guardrails_coverage.xml - # - guardrails_coverage - - # google_generate_content_endpoint_testing: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "respx==0.22.0" - # pip install "pydantic==2.10.2" - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml google_generate_content_endpoint_coverage.xml - # mv .coverage google_generate_content_endpoint_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - google_generate_content_endpoint_coverage.xml - # - google_generate_content_endpoint_coverage - - # llm_responses_api_testing: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "respx==0.22.0" - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/llm_responses_api_testing --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml llm_responses_api_coverage.xml - # mv .coverage llm_responses_api_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - llm_responses_api_coverage.xml - # - llm_responses_api_coverage - # ocr_testing: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "respx==0.22.0" - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml ocr_coverage.xml - # mv .coverage ocr_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - ocr_coverage.xml - # - ocr_coverage - # search_testing: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "respx==0.22.0" - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml search_coverage.xml - # mv .coverage search_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - search_coverage.xml - # - search_coverage - # # Split litellm_mapped_tests into 3 parallel jobs for 3x faster execution - # litellm_mapped_tests_proxy: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - # resource_class: xlarge - # steps: - # - setup_litellm_test_deps - # - run: - # name: Run proxy tests - # command: | - # prisma generate - # python -m pytest tests/test_litellm/proxy --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml litellm_proxy_tests_coverage.xml - # mv .coverage litellm_proxy_tests_coverage - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - litellm_proxy_tests_coverage.xml - # - litellm_proxy_tests_coverage - # litellm_mapped_tests_llms: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - # resource_class: xlarge - # steps: - # - setup_litellm_test_deps - # - run: - # name: Run LLM provider tests - # command: | - # python -m pytest tests/test_litellm/llms --cov=litellm --cov-report=xml --junitxml=test-results/junit-llms.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml litellm_llms_tests_coverage.xml - # mv .coverage litellm_llms_tests_coverage - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - litellm_llms_tests_coverage.xml - # - litellm_llms_tests_coverage - # litellm_mapped_tests_core: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - # resource_class: xlarge - # steps: - # - setup_litellm_test_deps - # - run: - # name: Run core tests - # command: | - # python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml litellm_core_tests_coverage.xml - # mv .coverage litellm_core_tests_coverage - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - litellm_core_tests_coverage.xml - # - litellm_core_tests_coverage - # litellm_mapped_tests_litellm_core_utils: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - # resource_class: xlarge - # steps: - # - setup_litellm_test_deps - # - run: - # name: Run litellm_core_utils tests - # command: | - # python -m pytest tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-litellm-core-utils.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml litellm_core_utils_tests_coverage.xml - # mv .coverage litellm_core_utils_tests_coverage - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - litellm_core_utils_tests_coverage.xml - # - litellm_core_utils_tests_coverage - # litellm_mapped_tests_integrations: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - # resource_class: xlarge - # steps: - # - setup_litellm_test_deps - # - run: - # name: Run integrations tests - # command: | - # python -m pytest tests/test_litellm/integrations --cov=litellm --cov-report=xml --junitxml=test-results/junit-integrations.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml litellm_integrations_tests_coverage.xml - # mv .coverage litellm_integrations_tests_coverage - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - litellm_integrations_tests_coverage.xml - # - litellm_integrations_tests_coverage - # litellm_mapped_enterprise_tests: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest-mock==3.12.0" - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "respx==0.22.0" - # pip install "hypercorn==0.17.3" - # pip install "pydantic==2.11.0" - # pip install "mcp==1.25.0" - # pip install "requests-mock>=1.12.1" - # pip install "responses==0.25.7" - # pip install "pytest-xdist==3.6.1" - # pip install "semantic_router==0.1.10" - # pip install "fastapi-offline==1.7.3" - # - setup_litellm_enterprise_pip - # - run: - # name: Run enterprise tests - # command: | - # pwd - # ls - # prisma generate - # python -m pytest -vv tests/enterprise --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit-enterprise.xml --durations=10 -n 8 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml litellm_mapped_tests_coverage.xml - # mv .coverage litellm_mapped_tests_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - litellm_mapped_tests_coverage.xml - # - litellm_mapped_tests_coverage - # batches_testing: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "respx==0.22.0" - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-asyncio==0.21.1" - # pip install "pytest-cov==5.0.0" - # pip install "google-generativeai==0.3.2" - # pip install "google-cloud-aiplatform==1.43.0" - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml batches_coverage.xml - # mv .coverage batches_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - batches_coverage.xml - # - batches_coverage - # litellm_utils_testing: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # pip install numpydoc - # python -m pip install -r requirements.txt - # pip install "respx==0.22.0" - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-asyncio==0.21.1" - # pip install "pytest-cov==5.0.0" - # pip install "google-generativeai==0.3.2" - # pip install "google-cloud-aiplatform==1.43.0" - # pip install pytest-mock - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml litellm_utils_coverage.xml - # mv .coverage litellm_utils_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - litellm_utils_coverage.xml - # - litellm_utils_coverage - - # pass_through_unit_testing: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "respx==0.22.0" - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml pass_through_unit_tests_coverage.xml - # mv .coverage pass_through_unit_tests_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - pass_through_unit_tests_coverage.xml - # - pass_through_unit_tests_coverage - # image_gen_testing: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "respx==0.22.0" - # pip install "pytest-xdist==3.6.1" - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/image_gen_tests -n 4 --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml image_gen_coverage.xml - # mv .coverage image_gen_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - image_gen_coverage.xml - # - image_gen_coverage - # logging_testing: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-asyncio==0.21.1" - # pip install pytest-mock - # pip install "respx==0.22.0" - # pip install "google-generativeai==0.3.2" - # pip install "google-cloud-aiplatform==1.43.0" - # pip install "mlflow==2.17.2" - # pip install "anthropic==0.52.0" - # pip install "blockbuster==1.5.24" - # pip install "pytest-xdist==3.6.1" - # # Run pytest and generate JUnit XML report - # - setup_litellm_enterprise_pip - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/logging_callback_tests --cov=litellm -n 4 --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml logging_coverage.xml - # mv .coverage logging_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - logging_coverage.xml - # - logging_coverage - # audio_testing: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-cov==5.0.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "respx==0.22.0" - # # Run pytest and generate JUnit XML report - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Rename the coverage files - # command: | - # mv coverage.xml audio_coverage.xml - # mv .coverage audio_coverage - - # # Store test results - # - store_test_results: - # path: test-results - # - persist_to_workspace: - # root: . - # paths: - # - audio_coverage.xml - # - audio_coverage - # installing_litellm_on_python: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # pip install python-dotenv - # pip install pytest - # pip install tiktoken - # pip install aiohttp - # pip install openai - # pip install click - # pip install "boto3==1.36.0" - # pip install jinja2 - # pip install "tokenizers==0.20.0" - # pip install "uvloop==0.21.0" - # pip install "fastuuid==0.12.0" - # pip install jsonschema - # - setup_litellm_enterprise_pip - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/local_testing/test_basic_python_version.py - - # installing_litellm_on_python_3_13: - # docker: - # - image: cimg/python:3.13.1 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # python -m pip install wheel setuptools - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-asyncio==0.21.1" - # pip install "pytest-cov==5.0.0" - # pip install "tomli==2.2.1" - # pip install "mcp==1.25.0" - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/local_testing/test_basic_python_version.py - # helm_chart_testing: - # machine: - # image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker - # resource_class: medium - # working_directory: ~/project - - # steps: - # - checkout - # - setup_google_dns - # # Install Helm - # - run: - # name: Install Helm - # command: | - # curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - - # # Install kind - # - run: - # name: Install Kind - # command: | - # curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - # chmod +x ./kind - # sudo mv ./kind /usr/local/bin/kind - - # # Install kubectl - # - run: - # name: Install kubectl - # command: | - # curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - # chmod +x kubectl - # sudo mv kubectl /usr/local/bin/ - - # # Create kind cluster - # - run: - # name: Create Kind Cluster - # command: | - # kind create cluster --name litellm-test - - # - run: - # name: Build Docker image for helm tests - # command: | - # IMAGE_TAG=${CIRCLE_SHA1:-ci} - # docker build -t litellm-ci:${IMAGE_TAG} -f docker/Dockerfile.database . - - # - run: - # name: Load Docker image into Kind - # command: | - # IMAGE_TAG=${CIRCLE_SHA1:-ci} - # kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test - - # # Run helm lint - # - run: - # name: Run helm lint - # command: | - # helm lint ./deploy/charts/litellm-helm - - # # Run helm tests - # - run: - # name: Run helm tests - # command: | - # IMAGE_TAG=${CIRCLE_SHA1:-ci} - # helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \ - # --set image.repository=litellm-ci \ - # --set image.tag=${IMAGE_TAG} \ - # --set image.pullPolicy=Never - # # Wait for pod to be ready - # echo "Waiting 30 seconds for pod to be ready..." - # sleep 30 - - # # Print pod logs before running tests - # echo "Printing pod logs..." - # kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}") - - # # Run the helm tests - # helm test litellm --logs - # helm test litellm --logs - - # # Cleanup - # - run: - # name: Cleanup - # command: | - # kind delete cluster --name litellm-test - # when: always # This ensures cleanup runs even if previous steps fail - - # check_code_and_doc_quality: - # docker: - # - image: cimg/python:3.11 - # auth: - # username: ${DOCKERHUB_USERNAME} - # password: ${DOCKERHUB_PASSWORD} - # working_directory: ~/project/litellm - - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Dependencies - # command: | - # python -m pip install --upgrade pip - # pip install ruff - # pip install pylint - # pip install pyright - # pip install beautifulsoup4 - # pip install . - # curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - # - run: python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) - # - run: ruff check ./litellm - # # - run: python ./tests/documentation_tests/test_general_setting_keys.py - # - run: python ./tests/code_coverage_tests/check_licenses.py - # - run: python ./tests/code_coverage_tests/check_provider_folders_documented.py - # - run: python ./tests/code_coverage_tests/router_code_coverage.py - # - run: python ./tests/code_coverage_tests/test_chat_completion_imports.py - # - run: python ./tests/code_coverage_tests/info_log_check.py - # - run: python ./tests/code_coverage_tests/test_ban_set_verbose.py - # - run: python ./tests/code_coverage_tests/code_qa_check_tests.py - # - run: python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py - # - run: python ./tests/code_coverage_tests/test_proxy_types_import.py - # - run: python ./tests/code_coverage_tests/callback_manager_test.py - # - run: python ./tests/code_coverage_tests/recursive_detector.py - # - run: python ./tests/code_coverage_tests/test_router_strategy_async.py - # - run: python ./tests/code_coverage_tests/litellm_logging_code_coverage.py - # - run: python ./tests/documentation_tests/test_env_keys.py - # - run: python ./tests/documentation_tests/test_router_settings.py - # - run: python ./tests/documentation_tests/test_api_docs.py - # - run: python ./tests/code_coverage_tests/ensure_async_clients_test.py - # - run: python ./tests/code_coverage_tests/enforce_llms_folder_style.py - # - run: python ./tests/documentation_tests/test_circular_imports.py - # - run: python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py - # - run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py - # - run: python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py - # - run: python ./tests/code_coverage_tests/check_fastuuid_usage.py - # - run: python ./tests/code_coverage_tests/memory_test.py - # - run: helm lint ./deploy/charts/litellm-helm - - # db_migration_disable_update_check: - # machine: - # image: ubuntu-2204:2023.10.1 - # resource_class: xlarge - # working_directory: ~/project - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Python 3.9 - # command: | - # curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - # bash miniconda.sh -b -p $HOME/miniconda - # export PATH="$HOME/miniconda/bin:$PATH" - # conda init bash - # source ~/.bashrc - # conda create -n myenv python=3.9 -y - # conda activate myenv - # python --version - # - run: - # name: Install Dependencies - # command: | - # pip install "pytest==7.3.1" - # pip install "pytest-asyncio==0.21.1" - # pip install aiohttp - # pip install apscheduler - # - attach_workspace: - # at: ~/project - # - run: - # name: Load Docker Database Image - # command: | - # gunzip -c litellm-docker-database.tar.gz | docker load - # docker images | grep litellm-docker-database - # - run: - # name: Run Docker container - # command: | - # docker run -d \ - # -p 4000:4000 \ - # -e DATABASE_URL=$PROXY_DATABASE_URL \ - # -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ - # -e DISABLE_SCHEMA_UPDATE="True" \ - # -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ - # -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ - # -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ - # --name my-app \ - # litellm-docker-database:ci \ - # --config /app/config.yaml \ - # --port 4000 - # - run: - # name: Install curl and dockerize - # command: | - # sudo apt-get update - # sudo apt-get install -y curl - # sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - # sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - # sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - # - run: - # name: Wait for container to be ready - # command: dockerize -wait http://localhost:4000 -timeout 1m - # - run: - # name: Check container logs for expected message - # command: | - # echo "=== Printing Full Container Startup Logs ===" - # LOG_OUTPUT="$(docker logs my-app 2>&1)" - # printf '%s\n' "$LOG_OUTPUT" - # echo "=== End of Full Container Startup Logs ===" - - # if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then - # echo "Expected message found in logs. Test passed." - # else - # echo "Expected message not found in logs. Test failed." - # exit 1 - # fi - # - run: - # name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) - # command: | - # python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 - # no_output_timeout: 120m - - # build_and_test: - # machine: - # image: ubuntu-2204:2023.10.1 - # resource_class: xlarge - # working_directory: ~/project - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Docker CLI (In case it's not already installed) - # command: | - # curl -fsSL https://get.docker.com | sh - # sudo usermod -aG docker $USER - # docker version - # - run: - # name: Install Python 3.9 - # command: | - # curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - # bash miniconda.sh -b -p $HOME/miniconda - # export PATH="$HOME/miniconda/bin:$PATH" - # conda init bash - # source ~/.bashrc - # conda create -n myenv python=3.9 -y - # conda activate myenv - # python --version - # - run: - # name: Install Dependencies - # command: | - # pip install "pytest==7.3.1" - # pip install "pytest-asyncio==0.21.1" - # pip install aiohttp - # python -m pip install --upgrade pip - # python -m pip install -r .circleci/requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-mock==3.12.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "mypy==1.18.2" - # pip install "google-generativeai==0.3.2" - # pip install "google-cloud-aiplatform==1.43.0" - # pip install pyarrow - # pip install "boto3==1.36.0" - # pip install "aioboto3==13.4.0" - # pip install langchain - # pip install "langfuse>=2.0.0" - # pip install "logfire==0.29.0" - # pip install numpydoc - # pip install prisma - # pip install fastapi - # pip install jsonschema - # pip install "httpx==0.24.1" - # pip install "gunicorn==21.2.0" - # pip install "anyio==3.7.1" - # pip install "aiodynamo==23.10.1" - # pip install "asyncio==3.4.3" - # pip install "PyGithub==1.59.1" - # pip install "openai==1.100.1" - # pip install "litellm[proxy]" - # pip install "pytest-xdist==3.6.1" - # - run: - # name: Install dockerize - # command: | - # wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - # sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - # rm dockerize-linux-amd64-v0.6.1.tar.gz - # - run: - # name: Start PostgreSQL Database - # command: | - # docker run -d \ - # --name postgres-db \ - # -e POSTGRES_USER=postgres \ - # -e POSTGRES_PASSWORD=postgres \ - # -e POSTGRES_DB=circle_test \ - # -p 5432:5432 \ - # postgres:14 - # - run: - # name: Wait for PostgreSQL to be ready - # command: dockerize -wait tcp://localhost:5432 -timeout 1m - # - run: - # name: Build Docker image - # command: docker build -t my-app:latest -f ./docker/Dockerfile.database . - # - run: - # name: Run Docker container - # command: | - # docker run -d \ - # -p 4000:4000 \ - # -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - # -e USE_PRISMA_MIGRATE=True \ - # -e AZURE_API_KEY=$AZURE_API_KEY \ - # -e REDIS_HOST=$REDIS_HOST \ - # -e REDIS_PASSWORD=$REDIS_PASSWORD \ - # -e REDIS_PORT=$REDIS_PORT \ - # -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ - # -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ - # -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ - # -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ - # -e GROQ_API_KEY=$GROQ_API_KEY \ - # -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ - # -e COHERE_API_KEY=$COHERE_API_KEY \ - # -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ - # -e AWS_REGION_NAME=$AWS_REGION_NAME \ - # -e AUTO_INFER_REGION=True \ - # -e OPENAI_API_KEY=$OPENAI_API_KEY \ - # -e USE_DDTRACE=True \ - # -e DD_API_KEY=$DD_API_KEY \ - # -e DD_SITE=$DD_SITE \ - # -e LITELLM_LICENSE=$LITELLM_LICENSE \ - # -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ - # -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ - # -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ - # -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ - # --add-host host.docker.internal:host-gateway \ - # --name my-app \ - # -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \ - # my-app:latest \ - # --config /app/config.yaml \ - # --port 4000 \ - # --detailed_debug \ - # - run: - # name: Install curl - # command: | - # sudo apt-get update - # sudo apt-get install -y curl - # - run: - # name: Start outputting logs - # command: docker logs -f my-app - # background: true - # - run: - # name: Wait for app to be ready - # command: dockerize -wait http://localhost:4000 -timeout 5m - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests - # no_output_timeout: 120m - - # # Store test results - # - store_test_results: - # path: test-results - # e2e_openai_endpoints: - # machine: - # image: ubuntu-2204:2023.10.1 - # resource_class: xlarge - # working_directory: ~/project - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Docker CLI (In case it's not already installed) - # command: | - # curl -fsSL https://get.docker.com | sh - # sudo usermod -aG docker $USER - # docker version - # - run: - # name: Install Python 3.10 - # command: | - # curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - # bash miniconda.sh -b -p $HOME/miniconda - # export PATH="$HOME/miniconda/bin:$PATH" - # conda init bash - # source ~/.bashrc - # conda create -n myenv python=3.10 -y - # conda activate myenv - # python --version - # - run: - # name: Install Dependencies - # command: | - # pip install "pytest==7.3.1" - # pip install "pytest-asyncio==0.21.1" - # pip install aiohttp - # python -m pip install --upgrade pip - # python -m pip install -r .circleci/requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-mock==3.12.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "mypy==1.18.2" - # pip install "jsonlines==4.0.0" - # pip install "google-generativeai==0.3.2" - # pip install "google-cloud-aiplatform==1.43.0" - # pip install pyarrow - # pip install "boto3==1.36.0" - # pip install "aioboto3==13.4.0" - # pip install langchain - # pip install "langchain_mcp_adapters==0.0.5" - # pip install "langfuse>=2.0.0" - # pip install "logfire==0.29.0" - # pip install numpydoc - # pip install prisma - # pip install fastapi - # pip install jsonschema - # pip install "httpx==0.24.1" - # pip install "gunicorn==21.2.0" - # pip install "anyio==3.7.1" - # pip install "aiodynamo==23.10.1" - # pip install "asyncio==3.4.3" - # pip install "PyGithub==1.59.1" - # pip install "openai==1.100.1" - # # Run pytest and generate JUnit XML report - # - run: - # name: Install dockerize - # command: | - # wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - # sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - # rm dockerize-linux-amd64-v0.6.1.tar.gz - # - run: - # name: Start PostgreSQL Database - # command: | - # docker run -d \ - # --name postgres-db \ - # -e POSTGRES_USER=postgres \ - # -e POSTGRES_PASSWORD=postgres \ - # -e POSTGRES_DB=circle_test \ - # -p 5432:5432 \ - # postgres:14 - # - run: - # name: Wait for PostgreSQL to be ready - # command: dockerize -wait tcp://localhost:5432 -timeout 1m - # - attach_workspace: - # at: ~/project - # - run: - # name: Load Docker Database Image - # command: | - # gunzip -c litellm-docker-database.tar.gz | docker load - # docker images | grep litellm-docker-database - # - run: - # name: Run Docker container - # command: | - # docker run -d \ - # -p 4000:4000 \ - # -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - # -e AZURE_API_KEY=$AZURE_API_KEY \ - # -e AZURE_API_BASE=$AZURE_API_BASE \ - # -e AZURE_API_VERSION="2024-05-01-preview" \ - # -e REDIS_HOST=$REDIS_HOST \ - # -e REDIS_PASSWORD=$REDIS_PASSWORD \ - # -e REDIS_PORT=$REDIS_PORT \ - # -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ - # -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ - # -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ - # -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ - # -e GROQ_API_KEY=$GROQ_API_KEY \ - # -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ - # -e COHERE_API_KEY=$COHERE_API_KEY \ - # -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ - # -e AWS_REGION_NAME=$AWS_REGION_NAME \ - # -e AUTO_INFER_REGION=True \ - # -e USE_DDTRACE=True \ - # -e DD_API_KEY=$DD_API_KEY \ - # -e DD_SITE=$DD_SITE \ - # -e OPENAI_API_KEY=$OPENAI_API_KEY \ - # -e LITELLM_LICENSE=$LITELLM_LICENSE \ - # -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ - # -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ - # -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ - # -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ - # --add-host host.docker.internal:host-gateway \ - # --name my-app \ - # -v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \ - # litellm-docker-database:ci \ - # --config /app/config.yaml \ - # --port 4000 \ - # --detailed_debug \ - # - run: - # name: Install curl - # command: | - # sudo apt-get update - # sudo apt-get install -y curl - # - run: - # name: Start outputting logs - # command: docker logs -f my-app - # background: true - # - run: - # name: Wait for app to be ready - # command: dockerize -wait http://localhost:4000 -timeout 5m - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - - # # Store test results - # - store_test_results: - # path: test-results - # proxy_logging_guardrails_model_info_tests: - # machine: - # image: ubuntu-2204:2023.10.1 - # resource_class: xlarge - # working_directory: ~/project - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Docker CLI (In case it's not already installed) - # command: | - # curl -fsSL https://get.docker.com | sh - # sudo usermod -aG docker $USER - # docker version - # - run: - # name: Install Python 3.9 - # command: | - # curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - # bash miniconda.sh -b -p $HOME/miniconda - # export PATH="$HOME/miniconda/bin:$PATH" - # conda init bash - # source ~/.bashrc - # conda create -n myenv python=3.9 -y - # conda activate myenv - # python --version - # - run: - # name: Install Dependencies - # command: | - # pip install "pytest==7.3.1" - # pip install "pytest-asyncio==0.21.1" - # pip install aiohttp - # python -m pip install --upgrade pip - # python -m pip install -r .circleci/requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-mock==3.12.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "mypy==1.18.2" - # pip install "google-generativeai==0.3.2" - # pip install "google-cloud-aiplatform==1.43.0" - # pip install pyarrow - # pip install "boto3==1.36.0" - # pip install "aioboto3==13.4.0" - # pip install langchain - # pip install "langfuse>=2.0.0" - # pip install "logfire==0.29.0" - # pip install numpydoc - # pip install prisma - # pip install fastapi - # pip install jsonschema - # pip install "httpx==0.24.1" - # pip install "gunicorn==21.2.0" - # pip install "anyio==3.7.1" - # pip install "aiodynamo==23.10.1" - # pip install "asyncio==3.4.3" - # pip install "PyGithub==1.59.1" - # pip install "openai==1.100.1" - # - run: - # name: Install dockerize - # command: | - # wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - # sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - # rm dockerize-linux-amd64-v0.6.1.tar.gz - # - run: - # name: Start PostgreSQL Database - # command: | - # docker run -d \ - # --name postgres-db \ - # -e POSTGRES_USER=postgres \ - # -e POSTGRES_PASSWORD=postgres \ - # -e POSTGRES_DB=circle_test \ - # -p 5432:5432 \ - # postgres:14 - # - run: - # name: Wait for PostgreSQL to be ready - # command: dockerize -wait tcp://localhost:5432 -timeout 1m - # - attach_workspace: - # at: ~/project - # - run: - # name: Load Docker Database Image - # command: | - # gunzip -c litellm-docker-database.tar.gz | docker load - # docker images | grep litellm-docker-database - # - run: - # name: Run Docker container - # # intentionally give bad redis credentials here - # # the OTEL test - should get this as a trace - # command: | - # docker run -d \ - # -p 4000:4000 \ - # -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - # -e REDIS_HOST=$REDIS_HOST \ - # -e REDIS_PASSWORD=$REDIS_PASSWORD \ - # -e REDIS_PORT=$REDIS_PORT \ - # -e LITELLM_MASTER_KEY="sk-1234" \ - # -e OPENAI_API_KEY=$OPENAI_API_KEY \ - # -e LITELLM_LICENSE=$LITELLM_LICENSE \ - # -e OTEL_EXPORTER="in_memory" \ - # -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ - # -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ - # -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ - # -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ - # -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ - # -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ - # -e USE_DDTRACE=True \ - # -e DD_API_KEY=$DD_API_KEY \ - # -e DD_SITE=$DD_SITE \ - # -e AWS_REGION_NAME=$AWS_REGION_NAME \ - # -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ - # -e COHERE_API_KEY=$COHERE_API_KEY \ - # -e GCS_FLUSH_INTERVAL="1" \ - # --add-host host.docker.internal:host-gateway \ - # --name my-app \ - # -v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \ - # -v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \ - # litellm-docker-database:ci \ - # --config /app/config.yaml \ - # --port 4000 \ - # --detailed_debug \ - # - run: - # name: Install curl and dockerize - # command: | - # sudo apt-get update - # sudo apt-get install -y curl - # sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - # sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - # sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - # - run: - # name: Start outputting logs - # command: docker logs -f my-app - # background: true - # - run: - # name: Wait for app to be ready - # command: dockerize -wait http://localhost:4000 -timeout 5m - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: - # 120m - # # Clean up first container - # - run: - # name: Stop and remove first container - # command: | - # docker stop my-app - # docker rm my-app - - # # Second Docker Container Run with Different Config - # # NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license - # - run: - # name: Run Second Docker container - # command: | - # docker run -d \ - # -p 4000:4000 \ - # -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - # -e REDIS_HOST=$REDIS_HOST \ - # -e REDIS_PASSWORD=$REDIS_PASSWORD \ - # -e REDIS_PORT=$REDIS_PORT \ - # -e LITELLM_MASTER_KEY="sk-1234" \ - # -e OPENAI_API_KEY=$OPENAI_API_KEY \ - # -e LITELLM_LICENSE="bad-license" \ - # --add-host host.docker.internal:host-gateway \ - # --name my-app-3 \ - # -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \ - # litellm-docker-database:ci \ - # --config /app/config.yaml \ - # --port 4000 \ - # --detailed_debug - - # - run: - # name: Start outputting logs for second container - # command: docker logs -f my-app-2 - # background: true - - # - run: - # name: Wait for second app to be ready - # command: dockerize -wait http://localhost:4000 -timeout 5m - - # - run: - # name: Run second round of tests - # command: | - # python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 - # no_output_timeout: 120m - - # # Store test results - # - store_test_results: - # path: test-results - # proxy_spend_accuracy_tests: - # machine: - # image: ubuntu-2204:2023.10.1 - # resource_class: xlarge - # working_directory: ~/project - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Docker CLI (In case it's not already installed) - # command: | - # curl -fsSL https://get.docker.com | sh - # sudo usermod -aG docker $USER - # docker version - # - run: - # name: Install Python 3.9 - # command: | - # curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - # bash miniconda.sh -b -p $HOME/miniconda - # export PATH="$HOME/miniconda/bin:$PATH" - # conda init bash - # source ~/.bashrc - # conda create -n myenv python=3.9 -y - # conda activate myenv - # python --version - # - run: - # name: Install Dependencies - # command: | - # pip install "pytest==7.3.1" - # pip install "pytest-asyncio==0.21.1" - # pip install aiohttp - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # - run: - # name: Install dockerize - # command: | - # wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - # sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - # rm dockerize-linux-amd64-v0.6.1.tar.gz - # - run: - # name: Start PostgreSQL Database - # command: | - # docker run -d \ - # --name postgres-db \ - # -e POSTGRES_USER=postgres \ - # -e POSTGRES_PASSWORD=postgres \ - # -e POSTGRES_DB=circle_test \ - # -p 5432:5432 \ - # postgres:14 - # - run: - # name: Wait for PostgreSQL to be ready - # command: dockerize -wait tcp://localhost:5432 -timeout 1m - # - attach_workspace: - # at: ~/project - # - run: - # name: Load Docker Database Image - # command: | - # gunzip -c litellm-docker-database.tar.gz | docker load - # docker images | grep litellm-docker-database - # - run: - # name: Run Docker container - # # intentionally give bad redis credentials here - # # the OTEL test - should get this as a trace - # command: | - # docker run -d \ - # -p 4000:4000 \ - # -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - # -e REDIS_HOST=$REDIS_HOST \ - # -e REDIS_PASSWORD=$REDIS_PASSWORD \ - # -e REDIS_PORT=$REDIS_PORT \ - # -e LITELLM_MASTER_KEY="sk-1234" \ - # -e OPENAI_API_KEY=$OPENAI_API_KEY \ - # -e LITELLM_LICENSE=$LITELLM_LICENSE \ - # -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ - # -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ - # -e USE_DDTRACE=True \ - # -e DD_API_KEY=$DD_API_KEY \ - # -e DD_SITE=$DD_SITE \ - # -e AWS_REGION_NAME=$AWS_REGION_NAME \ - # --add-host host.docker.internal:host-gateway \ - # --name my-app \ - # -v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \ - # litellm-docker-database:ci \ - # --config /app/config.yaml \ - # --port 4000 \ - # --detailed_debug \ - # - run: - # name: Install curl - # command: | - # sudo apt-get update - # sudo apt-get install -y curl - # - run: - # name: Start outputting logs - # command: docker logs -f my-app - # background: true - # - run: - # name: Wait for app to be ready - # command: dockerize -wait http://localhost:4000 -timeout 5m - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: - # 120m - # # Clean up first container - # - run: - # name: Stop and remove first container - # command: | - # docker stop my-app - # docker rm my-app - - # proxy_multi_instance_tests: - # machine: - # image: ubuntu-2204:2023.10.1 - # resource_class: xlarge - # working_directory: ~/project - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Docker CLI (In case it's not already installed) - # command: | - # curl -fsSL https://get.docker.com | sh - # sudo usermod -aG docker $USER - # docker version - # - run: - # name: Install Python 3.9 - # command: | - # curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - # bash miniconda.sh -b -p $HOME/miniconda - # export PATH="$HOME/miniconda/bin:$PATH" - # conda init bash - # source ~/.bashrc - # conda create -n myenv python=3.9 -y - # conda activate myenv - # python --version - # - run: - # name: Install Dependencies - # command: | - # pip install "pytest==7.3.1" - # pip install "pytest-asyncio==0.21.1" - # pip install aiohttp - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-mock==3.12.0" - # pip install "pytest-asyncio==0.21.1" - # - run: - # name: Install dockerize - # command: | - # wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - # sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - # rm dockerize-linux-amd64-v0.6.1.tar.gz - # - run: - # name: Start PostgreSQL Database - # command: | - # docker run -d \ - # --name postgres-db \ - # -e POSTGRES_USER=postgres \ - # -e POSTGRES_PASSWORD=postgres \ - # -e POSTGRES_DB=circle_test \ - # -p 5432:5432 \ - # postgres:14 - # - run: - # name: Wait for PostgreSQL to be ready - # command: dockerize -wait tcp://localhost:5432 -timeout 1m - # - attach_workspace: - # at: ~/project - # - run: - # name: Load Docker Database Image - # command: | - # gunzip -c litellm-docker-database.tar.gz | docker load - # docker images | grep litellm-docker-database - # - run: - # name: Run Docker container 1 - # # intentionally give bad redis credentials here - # # the OTEL test - should get this as a trace - # command: | - # docker run -d \ - # -p 4000:4000 \ - # -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - # -e REDIS_HOST=$REDIS_HOST \ - # -e REDIS_PASSWORD=$REDIS_PASSWORD \ - # -e REDIS_PORT=$REDIS_PORT \ - # -e LITELLM_MASTER_KEY="sk-1234" \ - # -e LITELLM_LICENSE=$LITELLM_LICENSE \ - # -e USE_DDTRACE=True \ - # -e DD_API_KEY=$DD_API_KEY \ - # -e DD_SITE=$DD_SITE \ - # --add-host host.docker.internal:host-gateway \ - # --name my-app \ - # -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ - # litellm-docker-database:ci \ - # --config /app/config.yaml \ - # --port 4000 \ - # --detailed_debug \ - # - run: - # name: Run Docker container 2 - # command: | - # docker run -d \ - # -p 4001:4001 \ - # -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - # -e REDIS_HOST=$REDIS_HOST \ - # -e REDIS_PASSWORD=$REDIS_PASSWORD \ - # -e REDIS_PORT=$REDIS_PORT \ - # -e LITELLM_MASTER_KEY="sk-1234" \ - # -e LITELLM_LICENSE=$LITELLM_LICENSE \ - # -e USE_DDTRACE=True \ - # -e DD_API_KEY=$DD_API_KEY \ - # -e DD_SITE=$DD_SITE \ - # --add-host host.docker.internal:host-gateway \ - # --name my-app-2 \ - # -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ - # litellm-docker-database:ci \ - # --config /app/config.yaml \ - # --port 4001 \ - # --detailed_debug - # - run: - # name: Install curl and dockerize - # command: | - # sudo apt-get update - # sudo apt-get install -y curl - # sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - # sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - # sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - # - run: - # name: Start outputting logs - # command: docker logs -f my-app - # background: true - # - run: - # name: Wait for instance 1 to be ready - # command: dockerize -wait http://localhost:4000 -timeout 5m - # - run: - # name: Wait for instance 2 to be ready - # command: dockerize -wait http://localhost:4001 -timeout 5m - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: - # 120m - # # Clean up first container - # # Store test results - # - store_test_results: - # path: test-results - - # proxy_store_model_in_db_tests: - # machine: - # image: ubuntu-2204:2023.10.1 - # resource_class: xlarge - # working_directory: ~/project - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Docker CLI (In case it's not already installed) - # command: | - # curl -fsSL https://get.docker.com | sh - # sudo usermod -aG docker $USER - # docker version - # sudo systemctl restart docker - # - run: - # name: Install Python 3.9 - # command: | - # curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - # bash miniconda.sh -b -p $HOME/miniconda - # export PATH="$HOME/miniconda/bin:$PATH" - # conda init bash - # source ~/.bashrc - # conda create -n myenv python=3.9 -y - # conda activate myenv - # python --version - # - run: - # name: Install Dependencies - # command: | - # pip install "pytest==7.3.1" - # pip install "pytest-asyncio==0.21.1" - # pip install aiohttp - # python -m pip install --upgrade pip - # python -m pip install -r requirements.txt - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-mock==3.12.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "assemblyai==0.37.0" - # - run: - # name: Install dockerize - # command: | - # wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - # sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - # rm dockerize-linux-amd64-v0.6.1.tar.gz - # - run: - # name: Start PostgreSQL Database - # command: | - # docker run -d \ - # --name postgres-db \ - # -e POSTGRES_USER=postgres \ - # -e POSTGRES_PASSWORD=postgres \ - # -e POSTGRES_DB=circle_test \ - # -p 5432:5432 \ - # postgres:14 - # - run: - # name: Wait for PostgreSQL to be ready - # command: dockerize -wait tcp://localhost:5432 -timeout 1m - # - attach_workspace: - # at: ~/project - # - run: - # name: Load Docker Database Image - # command: | - # gunzip -c litellm-docker-database.tar.gz | docker load - # docker images | grep litellm-docker-database - # - run: - # name: Run Docker container - # # intentionally give bad redis credentials here - # # the OTEL test - should get this as a trace - # command: | - # docker run -d \ - # -p 4000:4000 \ - # -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - # -e STORE_MODEL_IN_DB="True" \ - # -e LITELLM_MASTER_KEY="sk-1234" \ - # -e LITELLM_LICENSE=$LITELLM_LICENSE \ - # --add-host host.docker.internal:host-gateway \ - # --name my-app \ - # -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \ - # litellm-docker-database:ci \ - # --config /app/config.yaml \ - # --port 4000 \ - # --detailed_debug \ - # - run: - # name: Install curl and dockerize - # command: | - # sudo apt-get update - # sudo apt-get install -y curl - # sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - # sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - # sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - # - run: - # name: Start outputting logs - # command: docker logs -f my-app - # background: true - # - run: - # name: Wait for app to be ready - # command: dockerize -wait http://localhost:4000 -timeout 5m - # - run: - # name: Run tests - # command: | - # pwd - # ls - # python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - # - run: - # name: Stop and remove containers - # command: | - # docker stop my-app || true - # docker rm my-app || true - # docker stop postgres-db || true - # docker rm postgres-db || true - # when: always - # - store_test_results: - # path: test-results - - # proxy_build_from_pip_tests: - # # Change from docker to machine executor - # machine: - # image: ubuntu-2204:2023.10.1 - # resource_class: xlarge - # working_directory: ~/project - # steps: - # - checkout - # - setup_google_dns - # # Remove Docker CLI installation since it's already available in machine executor - # - run: - # name: Install Python 3.13 - # command: | - # curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - # bash miniconda.sh -b -p $HOME/miniconda - # export PATH="$HOME/miniconda/bin:$PATH" - # conda init bash - # source ~/.bashrc - # conda create -n myenv python=3.13 -y - # conda activate myenv - # python --version - # - run: - # name: Install Dependencies - # command: | - # pip install "pytest==7.3.1" - # pip install "pytest-asyncio==0.21.1" - # pip install aiohttp - # python -m pip install --upgrade pip - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-mock==3.12.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "mypy==1.18.2" - # pip install apscheduler - # - run: - # name: Build Docker image - # command: | - # docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip . - # - run: - # name: Run Docker container - # # intentionally give bad redis credentials here - # # the OTEL test - should get this as a trace - # command: | - # docker run -d \ - # -p 4000:4000 \ - # -e DATABASE_URL=$PROXY_DATABASE_URL \ - # -e REDIS_HOST=$REDIS_HOST \ - # -e REDIS_PASSWORD=$REDIS_PASSWORD \ - # -e REDIS_PORT=$REDIS_PORT \ - # -e LITELLM_MASTER_KEY="sk-1234" \ - # -e OPENAI_API_KEY=$OPENAI_API_KEY \ - # -e LITELLM_LICENSE=$LITELLM_LICENSE \ - # -e OTEL_EXPORTER="in_memory" \ - # -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ - # -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ - # -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ - # -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ - # -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ - # -e AWS_REGION_NAME=$AWS_REGION_NAME \ - # -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ - # -e COHERE_API_KEY=$COHERE_API_KEY \ - # -e USE_DDTRACE=True \ - # -e DD_API_KEY=$DD_API_KEY \ - # -e DD_SITE=$DD_SITE \ - # -e GCS_FLUSH_INTERVAL="1" \ - # --name my-app \ - # -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \ - # my-app:latest \ - # --config /app/config.yaml \ - # --port 4000 \ - # --detailed_debug \ - # - run: - # name: Install curl and dockerize - # command: | - # sudo apt-get update - # sudo apt-get install -y curl - # sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - # sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - # sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - # - run: - # name: Start outputting logs - # command: docker logs -f my-app - # background: true - # - run: - # name: Wait for app to be ready - # command: dockerize -wait http://localhost:4000 -timeout 5m - # - run: - # name: Run tests - # command: | - # python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 - # no_output_timeout: - # 120m - # # Clean up first container - # - run: - # name: Stop and remove first container - # command: | - # docker stop my-app - # docker rm my-app - # proxy_pass_through_endpoint_tests: - # machine: - # image: ubuntu-2204:2023.10.1 - # resource_class: xlarge - # working_directory: ~/project - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Docker CLI (In case it's not already installed) - # command: | - # curl -fsSL https://get.docker.com | sh - # sudo usermod -aG docker $USER - # docker version - # - run: - # name: Install Python 3.10 - # command: | - # curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - # bash miniconda.sh -b -p $HOME/miniconda - # export PATH="$HOME/miniconda/bin:$PATH" - # conda init bash - # source ~/.bashrc - # conda create -n myenv python=3.10 -y - # conda activate myenv - # python --version - # - run: - # name: Install Dependencies - # command: | - # export PATH="$HOME/miniconda/bin:$PATH" - # source $HOME/miniconda/etc/profile.d/conda.sh - # conda activate myenv - # pip install "pytest==7.3.1" - # pip install "pytest-retry==1.6.3" - # pip install "pytest-asyncio==0.21.1" - # pip install "google-cloud-aiplatform==1.43.0" - # pip install aiohttp - # pip install "openai==1.100.1" - # pip install "assemblyai==0.37.0" - # python -m pip install --upgrade pip - # pip install "pydantic==2.10.2" - # pip install "pytest==7.3.1" - # pip install "pytest-mock==3.12.0" - # pip install "pytest-asyncio==0.21.1" - # pip install "boto3==1.36.0" - # pip install "mypy==1.18.2" - # pip install pyarrow - # pip install numpydoc - # pip install prisma - # pip install fastapi - # pip install jsonschema - # pip install "httpx==0.27.0" - # pip install "anyio==3.7.1" - # pip install "asyncio==3.4.3" - # pip install "PyGithub==1.59.1" - # pip install "google-cloud-aiplatform==1.59.0" - # pip install "anthropic==0.52.0" - # pip install "langchain_mcp_adapters==0.0.5" - # pip install "langchain_openai==0.2.1" - # pip install "langgraph==0.3.18" - # pip install "fastuuid==0.13.5" - # pip install -r requirements.txt - # - run: - # name: Install dockerize - # command: | - # wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - # sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - # rm dockerize-linux-amd64-v0.6.1.tar.gz - # - run: - # name: Start PostgreSQL Database - # command: | - # docker run -d \ - # --name postgres-db \ - # -e POSTGRES_USER=postgres \ - # -e POSTGRES_PASSWORD=postgres \ - # -e POSTGRES_DB=circle_test \ - # -p 5432:5432 \ - # postgres:14 - # - run: - # name: Wait for PostgreSQL to be ready - # command: dockerize -wait tcp://localhost:5432 -timeout 1m - # - attach_workspace: - # at: ~/project - # - run: - # name: Load Docker Database Image - # command: | - # gunzip -c litellm-docker-database.tar.gz | docker load - # docker images | grep litellm-docker-database - # - run: - # name: Run Docker container - # command: | - # docker run -d \ - # -p 4000:4000 \ - # -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - # -e LITELLM_MASTER_KEY="sk-1234" \ - # -e OPENAI_API_KEY=$OPENAI_API_KEY \ - # -e GEMINI_API_KEY=$GEMINI_API_KEY \ - # -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ - # -e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \ - # -e AZURE_API_KEY=$AZURE_API_KEY \ - # -e AZURE_API_BASE=$AZURE_API_BASE \ - # -e USE_DDTRACE=True \ - # -e DD_API_KEY=$DD_API_KEY \ - # -e DD_SITE=$DD_SITE \ - # -e LITELLM_LICENSE=$LITELLM_LICENSE \ - # --add-host host.docker.internal:host-gateway \ - # --name my-app \ - # -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \ - # -v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \ - # litellm-docker-database:ci \ - # --config /app/config.yaml \ - # --port 4000 \ - # --detailed_debug \ - # - run: - # name: Start outputting logs - # command: docker logs -f my-app - # background: true - # - run: - # name: Wait for app to be ready - # command: dockerize -wait http://localhost:4000 -timeout 5m - # # Add Ruby installation and testing before the existing Node.js and Python tests - # - run: - # name: Install Ruby and Bundler - # command: | - # # Import GPG keys first - # gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB || { - # curl -sSL https://rvm.io/mpapis.asc | gpg --import - - # curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - - # } - - # # Install Ruby version manager (RVM) - # curl -sSL https://get.rvm.io | bash -s stable - - # # Source RVM from the correct location - # source $HOME/.rvm/scripts/rvm - - # # Install Ruby 3.2.2 - # rvm install 3.2.2 - # rvm use 3.2.2 --default - - # # Install latest Bundler - # gem install bundler - - # - run: - # name: Run Ruby tests - # command: | - # source $HOME/.rvm/scripts/rvm - # cd tests/pass_through_tests/ruby_passthrough_tests - # bundle install - # bundle exec rspec - # no_output_timeout: 30m - # # New steps to run Node.js test - # - run: - # name: Install Node.js - # command: | - # export DEBIAN_FRONTEND=noninteractive - # curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash - - # sudo apt-get update - # sudo apt-get install -y nodejs - # node --version - # npm --version - - # - run: - # name: Install Node.js dependencies - # command: | - # npm install @google-cloud/vertexai - # npm install @google/generative-ai - # npm install --save-dev jest - - # - run: - # name: Run Vertex AI, Google AI Studio Node.js tests - # command: | - # npx jest tests/pass_through_tests --verbose - # no_output_timeout: 30m - # - run: - # name: Run tests - # command: | - # export PATH="$HOME/miniconda/bin:$PATH" - # source $HOME/miniconda/etc/profile.d/conda.sh - # conda activate myenv - # pwd - # ls - # python -m pytest -vv tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - - # # Store test results - # - store_test_results: - # path: test-results - - # proxy_e2e_anthropic_messages_tests: - # machine: - # image: ubuntu-2204:2023.10.1 - # resource_class: xlarge - # working_directory: ~/project - # steps: - # - checkout - # - setup_google_dns - # - run: - # name: Install Docker CLI (In case it's not already installed) - # command: | - # curl -fsSL https://get.docker.com | sh - # sudo usermod -aG docker $USER - # docker version - # - run: - # name: Install Python 3.10 - # command: | - # curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - # bash miniconda.sh -b -p $HOME/miniconda - # export PATH="$HOME/miniconda/bin:$PATH" - # conda init bash - # source ~/.bashrc - # conda create -n myenv python=3.10 -y - # conda activate myenv - # python --version - # - run: - # name: Install Dependencies - # command: | - # export PATH="$HOME/miniconda/bin:$PATH" - # source $HOME/miniconda/etc/profile.d/conda.sh - # conda activate myenv - # pip install "pytest==7.3.1" - # pip install "pytest-asyncio==0.21.1" - # pip install "boto3==1.36.0" - # pip install "httpx==0.27.0" - # pip install "claude-agent-sdk" - # pip install -r requirements.txt - # - run: - # name: Install dockerize - # command: | - # wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - # sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - # rm dockerize-linux-amd64-v0.6.1.tar.gz - # - run: - # name: Start PostgreSQL Database - # command: | - # docker run -d \ - # --name postgres-db \ - # -e POSTGRES_USER=postgres \ - # -e POSTGRES_PASSWORD=postgres \ - # -e POSTGRES_DB=circle_test \ - # -p 5432:5432 \ - # postgres:14 - # - run: - # name: Wait for PostgreSQL to be ready - # command: dockerize -wait tcp://localhost:5432 -timeout 1m - # - attach_workspace: - # at: ~/project - # - run: - # name: Load Docker Database Image - # command: | - # gunzip -c litellm-docker-database.tar.gz | docker load - # docker images | grep litellm-docker-database - # - run: - # name: Run Docker container with test config - # command: | - # docker run -d \ - # -p 4000:4000 \ - # -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - # -e LITELLM_MASTER_KEY="sk-1234" \ - # -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ - # -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ - # -e AWS_REGION_NAME="us-east-1" \ - # --add-host host.docker.internal:host-gateway \ - # --name my-app \ - # -v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \ - # litellm-docker-database:ci \ - # --config /app/config.yaml \ - # --port 4000 \ - # --detailed_debug - # - run: - # name: Start outputting logs - # command: docker logs -f my-app - # background: true - # - run: - # name: Wait for app to be ready - # command: dockerize -wait http://localhost:4000 -timeout 5m - # - run: - # name: Run Claude Agent SDK E2E Tests - # command: | - # export PATH="$HOME/miniconda/bin:$PATH" - # source $HOME/miniconda/etc/profile.d/conda.sh - # conda activate myenv - # export LITELLM_PROXY_URL="http://localhost:4000" - # export LITELLM_API_KEY="sk-1234" - # pwd - # ls - # python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5 - # no_output_timeout: 120m - - # # Store test results - # - store_test_results: - # path: test-results + local_testing_part1: + docker: + - image: cimg/python:3.12 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + parallelism: 4 + steps: + - checkout + - setup_google_dns + - run: + name: Show git commit hash + command: | + echo "Git commit hash: $CIRCLE_SHA1" + + - restore_cache: + keys: + - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r .circleci/requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install "pytest-cov==5.0.0" + pip install "mypy==1.18.2" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install pyarrow + pip install "boto3==1.36.0" + pip install "aioboto3==13.4.0" + pip install langchain + pip install lunary==0.2.5 + pip install "azure-identity==1.16.1" + pip install "langfuse==2.59.7" + pip install "logfire==0.29.0" + pip install numpydoc + pip install traceloop-sdk==0.21.1 + pip install opentelemetry-api==1.25.0 + pip install opentelemetry-sdk==1.25.0 + pip install opentelemetry-exporter-otlp==1.25.0 + pip install openai==1.100.1 + pip install prisma==0.11.0 + pip install "detect_secrets==1.5.0" + pip install "httpx==0.24.1" + pip install "respx==0.22.0" + pip install fastapi + pip install "gunicorn==21.2.0" + pip install "anyio==4.2.0" + pip install "aiodynamo==23.10.1" + pip install "asyncio==3.4.3" + pip install "apscheduler==3.10.4" + pip install "PyGithub==1.59.1" + pip install argon2-cffi + pip install "pytest-mock==3.12.0" + pip install python-multipart + pip install google-cloud-aiplatform + pip install prometheus-client==0.20.0 + pip install "pydantic==2.10.2" + pip install "diskcache==5.6.1" + pip install "Pillow==10.3.0" + pip install "jsonschema==4.22.0" + pip install "pytest-xdist==3.6.1" + pip install "pytest-timeout==2.2.0" + pip install "websockets==13.1.0" + pip install semantic_router --no-deps + pip install aurelio_sdk --no-deps + pip uninstall posthog -y + - setup_litellm_enterprise_pip + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Run prisma ./docker/entrypoint.sh + command: | + set +e + chmod +x docker/entrypoint.sh + ./docker/entrypoint.sh + set -e + - run: + name: Black Formatting + command: | + cd litellm + python -m pip install black + python -m black . + cd .. + + # Run pytest and generate JUnit XML report + - run: + name: Run tests (Part 1 - A-M) + command: | + mkdir test-results + + # Discover test files (A-M) + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py") + + echo "$TEST_FILES" | circleci tests run \ + --split-by=timings \ + --verbose \ + --command="xargs python -m pytest \ + -vv \ + --cov=litellm \ + --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=20 \ + -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ + -n 4 \ + --timeout=300 \ + --timeout_method=thread" + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml local_testing_part1_coverage.xml + mv .coverage local_testing_part1_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - local_testing_part1_coverage.xml + - local_testing_part1_coverage + local_testing_part2: + docker: + - image: cimg/python:3.12 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + parallelism: 4 + steps: + - checkout + - setup_google_dns + - run: + name: Show git commit hash + command: | + echo "Git commit hash: $CIRCLE_SHA1" + + - restore_cache: + keys: + - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r .circleci/requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install "pytest-cov==5.0.0" + pip install "mypy==1.18.2" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install pyarrow + pip install "boto3==1.36.0" + pip install "aioboto3==13.4.0" + pip install langchain + pip install lunary==0.2.5 + pip install "azure-identity==1.16.1" + pip install "langfuse==2.59.7" + pip install "logfire==0.29.0" + pip install numpydoc + pip install traceloop-sdk==0.21.1 + pip install opentelemetry-api==1.25.0 + pip install opentelemetry-sdk==1.25.0 + pip install opentelemetry-exporter-otlp==1.25.0 + pip install openai==1.100.1 + pip install prisma==0.11.0 + pip install "detect_secrets==1.5.0" + pip install "httpx==0.24.1" + pip install "respx==0.22.0" + pip install fastapi + pip install "gunicorn==21.2.0" + pip install "anyio==4.2.0" + pip install "aiodynamo==23.10.1" + pip install "asyncio==3.4.3" + pip install "apscheduler==3.10.4" + pip install "PyGithub==1.59.1" + pip install argon2-cffi + pip install "pytest-mock==3.12.0" + pip install python-multipart + pip install google-cloud-aiplatform + pip install prometheus-client==0.20.0 + pip install "pydantic==2.10.2" + pip install "diskcache==5.6.1" + pip install "Pillow==10.3.0" + pip install "jsonschema==4.22.0" + pip install "pytest-xdist==3.6.1" + pip install "pytest-timeout==2.2.0" + pip install "websockets==13.1.0" + pip install semantic_router --no-deps + pip install aurelio_sdk --no-deps + pip uninstall posthog -y + - setup_litellm_enterprise_pip + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Run prisma ./docker/entrypoint.sh + command: | + set +e + chmod +x docker/entrypoint.sh + ./docker/entrypoint.sh + set -e + - run: + name: Black Formatting + command: | + cd litellm + python -m pip install black + python -m black . + cd .. + + # Run pytest and generate JUnit XML report + - run: + name: Run tests (Part 2 - N-Z) + command: | + mkdir test-results + + # Discover test files (N-Z) + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py") + + echo "$TEST_FILES" | circleci tests run \ + --split-by=timings \ + --verbose \ + --command="xargs python -m pytest \ + -vv \ + --cov=litellm \ + --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=20 \ + -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ + -n 4 \ + --timeout=300 \ + --timeout_method=thread" + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml local_testing_part2_coverage.xml + mv .coverage local_testing_part2_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - local_testing_part2_coverage.xml + - local_testing_part2_coverage + langfuse_logging_unit_tests: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Show git commit hash + command: | + echo "Git commit hash: $CIRCLE_SHA1" + + - restore_cache: + keys: + - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r .circleci/requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install "pytest-cov==5.0.0" + pip install "mypy==1.18.2" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install pyarrow + pip install "boto3==1.36.0" + pip install "aioboto3==13.4.0" + pip install langchain + pip install lunary==0.2.5 + pip install "azure-identity==1.16.1" + pip install "langfuse==2.59.7" + pip install "logfire==0.29.0" + pip install numpydoc + pip install traceloop-sdk==0.21.1 + pip install opentelemetry-api==1.25.0 + pip install opentelemetry-sdk==1.25.0 + pip install opentelemetry-exporter-otlp==1.25.0 + pip install openai==1.100.1 + pip install prisma==0.11.0 + pip install "detect_secrets==1.5.0" + pip install "httpx==0.24.1" + pip install "respx==0.22.0" + pip install fastapi + pip install "gunicorn==21.2.0" + pip install "anyio==4.2.0" + pip install "aiodynamo==23.10.1" + pip install "asyncio==3.4.3" + pip install "apscheduler==3.10.4" + pip install "PyGithub==1.59.1" + pip install argon2-cffi + pip install "pytest-mock==3.12.0" + pip install python-multipart + pip install google-cloud-aiplatform + pip install prometheus-client==0.20.0 + pip install "pydantic==2.10.2" + pip install "diskcache==5.6.1" + pip install "Pillow==10.3.0" + pip install "jsonschema==4.22.0" + pip install "websockets==13.1.0" + - setup_litellm_enterprise_pip + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Run prisma ./docker/entrypoint.sh + command: | + set +e + chmod +x docker/entrypoint.sh + ./docker/entrypoint.sh + set -e + + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse" + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml langfuse_coverage.xml + mv .coverage langfuse_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - langfuse_coverage.xml + - langfuse_coverage + caching_unit_tests: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: DNS lookup for Redis host + command: | + sudo apt-get update + sudo apt-get install -y dnsutils + dig redis-19899.c239.us-east-1-2.ec2.redns.redis-cloud.com +short + - run: + name: Show git commit hash + command: | + echo "Git commit hash: $CIRCLE_SHA1" + + - restore_cache: + keys: + - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r .circleci/requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install "pytest-cov==5.0.0" + pip install "mypy==1.18.2" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install pyarrow + pip install "boto3==1.36.0" + pip install "aioboto3==13.4.0" + pip install langchain + pip install lunary==0.2.5 + pip install "azure-identity==1.16.1" + pip install "langfuse==2.59.7" + pip install "logfire==0.29.0" + pip install numpydoc + pip install traceloop-sdk==0.21.1 + pip install opentelemetry-api==1.25.0 + pip install opentelemetry-sdk==1.25.0 + pip install opentelemetry-exporter-otlp==1.25.0 + pip install openai==1.100.1 + pip install prisma==0.11.0 + pip install "detect_secrets==1.5.0" + pip install "httpx==0.24.1" + pip install "respx==0.22.0" + pip install fastapi + pip install "gunicorn==21.2.0" + pip install "anyio==4.2.0" + pip install "aiodynamo==23.10.1" + pip install "asyncio==3.4.3" + pip install "apscheduler==3.10.4" + pip install "PyGithub==1.59.1" + pip install argon2-cffi + pip install "pytest-mock==3.12.0" + pip install python-multipart + pip install google-cloud-aiplatform + pip install prometheus-client==0.20.0 + pip install "pydantic==2.10.2" + pip install "diskcache==5.6.1" + pip install "Pillow==10.3.0" + pip install "jsonschema==4.22.0" + pip install "websockets==13.1.0" + - setup_litellm_enterprise_pip + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Run prisma ./docker/entrypoint.sh + command: | + set +e + chmod +x docker/entrypoint.sh + ./docker/entrypoint.sh + set -e + + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "caching or cache" + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml caching_coverage.xml + mv .coverage caching_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - caching_coverage.xml + - caching_coverage + auth_ui_unit_tests: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install "pytest-cov==5.0.0" + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Run prisma ./docker/entrypoint.sh + command: | + set +e + chmod +x docker/entrypoint.sh + ./docker/entrypoint.sh + set -e + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/proxy_admin_ui_tests -x --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + + - run: + name: Rename the coverage files + command: | + mv coverage.xml auth_ui_unit_tests_coverage.xml + mv .coverage auth_ui_unit_tests_coverage + + # Store test results + - store_test_results: + path: test-results + + - persist_to_workspace: + root: . + paths: + - auth_ui_unit_tests_coverage.xml + - auth_ui_unit_tests_coverage + + litellm_router_testing: # Runs all tests with the "router" keyword + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "respx==0.22.0" + pip install "pytest-cov==5.0.0" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install "pytest-xdist==3.6.1" + pip install semantic_router --no-deps + pip install aurelio_sdk --no-deps + # Run pytest and generate JUnit XML report + - setup_litellm_enterprise_pip + - run: + name: Run tests + command: | + pwd + ls + python -m pytest tests/local_testing --cov=litellm --cov-report=xml -vv -k "router" -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_router_coverage.xml + mv .coverage litellm_router_coverage + # Store test results + - store_test_results: + path: test-results + + - persist_to_workspace: + root: . + paths: + - litellm_router_coverage.xml + - litellm_router_coverage + + litellm_router_unit_testing: # Runs all tests with the "router" keyword + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "respx==0.22.0" + pip install "pytest-cov==5.0.0" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install semantic_router --no-deps + pip install aurelio_sdk --no-deps + pip install "pytest-xdist==3.6.1" + # Run pytest and generate JUnit XML report + - setup_litellm_enterprise_pip + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/router_unit_tests --cov=litellm --cov-report=xml -x -s --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_router_unit_coverage.xml + mv .coverage litellm_router_unit_coverage + # Store test results + - store_test_results: + path: test-results + + - persist_to_workspace: + root: . + paths: + - litellm_router_unit_coverage.xml + - litellm_router_unit_coverage + litellm_security_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Show git commit hash + command: | + echo "Git commit hash: $CIRCLE_SHA1" + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + - run: + name: Install Python 3.13 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.13 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + python --version + which python + pip install --upgrade typing-extensions>=4.12.0 + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install aiohttp + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-mock==3.12.0" + pip install "pytest-asyncio==0.21.1" + pip install "mypy==1.18.2" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install pyarrow + pip install "boto3==1.36.0" + pip install "aioboto3==13.4.0" + pip install langchain + pip install "langfuse>=2.0.0" + pip install "logfire==0.29.0" + pip install numpydoc + pip install prisma + pip install fastapi + pip install jsonschema + pip install "httpx==0.24.1" + pip install "gunicorn==21.2.0" + pip install "anyio==3.7.1" + pip install "aiodynamo==23.10.1" + pip install "asyncio==3.4.3" + pip install "PyGithub==1.59.1" + pip install "openai==1.100.1" + pip install "pytest-cov==5.0.0" + pip install "apscheduler" + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - run: + name: Set DATABASE_URL environment variable + command: | + echo 'export DATABASE_URL="postgresql://postgres:postgres@localhost:5432/circle_test"' >> $BASH_ENV + source $BASH_ENV + - run: + name: Run Security Scans + command: | + chmod +x ci_cd/security_scans.sh + ./ci_cd/security_scans.sh + - run: + name: Run prisma ./docker/entrypoint.sh + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + set +e + chmod +x docker/entrypoint.sh + ./docker/entrypoint.sh + set -e + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + pwd + ls + python -m pytest tests/proxy_security_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_security_tests_coverage.xml + mv .coverage litellm_security_tests_coverage + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_security_tests_coverage.xml + - litellm_security_tests_coverage + # Split proxy unit tests into 3 jobs for faster execution and better debugging + # test_key_generate_prisma runs separately without parallel execution to avoid event loop issues with logging worker + litellm_proxy_unit_testing_key_generation: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: large + steps: + - checkout + - setup_google_dns + - run: + name: Show git commit hash + command: | + echo "Git commit hash: $CIRCLE_SHA1" + - run: + name: Install PostgreSQL + command: | + sudo apt-get update + sudo apt-get install -y postgresql-14 postgresql-contrib-14 + - restore_cache: + keys: + - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r .circleci/requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install "pytest-cov==5.0.0" + pip install "pytest-timeout==2.2.0" + pip install "pytest-forked==1.6.0" + pip install "mypy==1.18.2" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install "google-genai==1.22.0" + pip install pyarrow + pip install "boto3==1.36.0" + pip install "aioboto3==13.4.0" + pip install langchain + pip install lunary==0.2.5 + pip install "azure-identity==1.16.1" + pip install "langfuse==2.59.7" + pip install "logfire==0.29.0" + pip install numpydoc + pip install traceloop-sdk==0.21.1 + pip install opentelemetry-api==1.25.0 + pip install opentelemetry-sdk==1.25.0 + pip install opentelemetry-exporter-otlp==1.25.0 + pip install openai==1.100.1 + pip install prisma==0.11.0 + pip install "detect_secrets==1.5.0" + pip install "httpx==0.24.1" + pip install "respx==0.22.0" + pip install fastapi + pip install "gunicorn==21.2.0" + pip install "anyio==4.2.0" + pip install "aiodynamo==23.10.1" + pip install "asyncio==3.4.3" + pip install "apscheduler==3.10.4" + pip install "PyGithub==1.59.1" + pip install argon2-cffi + pip install "pytest-mock==3.12.0" + pip install python-multipart + pip install google-cloud-aiplatform + pip install prometheus-client==0.20.0 + pip install "pydantic==2.10.2" + pip install "diskcache==5.6.1" + pip install "Pillow==10.3.0" + pip install "jsonschema==4.22.0" + pip install "pytest-postgresql==7.0.1" + pip install "fakeredis==2.28.1" + - setup_litellm_enterprise_pip + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Run prisma ./docker/entrypoint.sh + command: | + set +e + chmod +x docker/entrypoint.sh + ./docker/entrypoint.sh + set -e + - run: + name: Run key generation tests (no parallel execution to avoid event loop issues) + command: | + pwd + ls + # Run without -n flag to avoid pytest-xdist event loop conflicts with logging worker + python -m pytest tests/proxy_unit_tests/test_key_generate_prisma.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-key-generation.xml --durations=10 --timeout=300 -vv --log-cli-level=INFO + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_proxy_unit_tests_key_generation_coverage.xml + mv .coverage litellm_proxy_unit_tests_key_generation_coverage + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_proxy_unit_tests_key_generation_coverage.xml + - litellm_proxy_unit_tests_key_generation_coverage + litellm_proxy_unit_testing_part1: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: large + steps: + - checkout + - setup_google_dns + - run: + name: Show git commit hash + command: | + echo "Git commit hash: $CIRCLE_SHA1" + - run: + name: Install PostgreSQL + command: | + sudo apt-get update + sudo apt-get install -y postgresql-14 postgresql-contrib-14 + - restore_cache: + keys: + - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r .circleci/requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install "pytest-cov==5.0.0" + pip install "pytest-timeout==2.2.0" + pip install "pytest-forked==1.6.0" + pip install "mypy==1.18.2" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install "google-genai==1.22.0" + pip install pyarrow + pip install "boto3==1.36.0" + pip install "aioboto3==13.4.0" + pip install langchain + pip install lunary==0.2.5 + pip install "azure-identity==1.16.1" + pip install "langfuse==2.59.7" + pip install "logfire==0.29.0" + pip install numpydoc + pip install traceloop-sdk==0.21.1 + pip install opentelemetry-api==1.25.0 + pip install opentelemetry-sdk==1.25.0 + pip install opentelemetry-exporter-otlp==1.25.0 + pip install openai==1.100.1 + pip install prisma==0.11.0 + pip install "detect_secrets==1.5.0" + pip install "httpx==0.24.1" + pip install "respx==0.22.0" + pip install fastapi + pip install "gunicorn==21.2.0" + pip install "anyio==4.2.0" + pip install "aiodynamo==23.10.1" + pip install "asyncio==3.4.3" + pip install "apscheduler==3.10.4" + pip install "PyGithub==1.59.1" + pip install argon2-cffi + pip install "pytest-mock==3.12.0" + pip install python-multipart + pip install google-cloud-aiplatform + pip install prometheus-client==0.20.0 + pip install "pydantic==2.10.2" + pip install "diskcache==5.6.1" + pip install "Pillow==10.3.0" + pip install "jsonschema==4.22.0" + pip install "pytest-postgresql==7.0.1" + pip install "fakeredis==2.28.1" + pip install "pytest-xdist==3.6.1" + - setup_litellm_enterprise_pip + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Run prisma ./docker/entrypoint.sh + command: | + set +e + chmod +x docker/entrypoint.sh + ./docker/entrypoint.sh + set -e + - run: + name: Run proxy unit tests (part 1 - auth checks only, key generation in separate job) + command: | + pwd + ls + # Run auth tests with parallel execution (test_key_generate_prisma moved to separate job to avoid event loop issues) + python -m pytest tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part1.xml --durations=10 -n 8 --timeout=300 -vv --log-cli-level=INFO + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_proxy_unit_tests_part1_coverage.xml + mv .coverage litellm_proxy_unit_tests_part1_coverage + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_proxy_unit_tests_part1_coverage.xml + - litellm_proxy_unit_tests_part1_coverage + litellm_proxy_unit_testing_part2: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: large + steps: + - checkout + - setup_google_dns + - run: + name: Show git commit hash + command: | + echo "Git commit hash: $CIRCLE_SHA1" + - run: + name: Install PostgreSQL + command: | + sudo apt-get update + sudo apt-get install -y postgresql-14 postgresql-contrib-14 + - restore_cache: + keys: + - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r .circleci/requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install "pytest-cov==5.0.0" + pip install "pytest-timeout==2.2.0" + pip install "pytest-forked==1.6.0" + pip install "mypy==1.18.2" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install "google-genai==1.22.0" + pip install pyarrow + pip install "boto3==1.36.0" + pip install "aioboto3==13.4.0" + pip install langchain + pip install lunary==0.2.5 + pip install "azure-identity==1.16.1" + pip install "langfuse==2.59.7" + pip install "logfire==0.29.0" + pip install numpydoc + pip install traceloop-sdk==0.21.1 + pip install opentelemetry-api==1.25.0 + pip install opentelemetry-sdk==1.25.0 + pip install opentelemetry-exporter-otlp==1.25.0 + pip install openai==1.100.1 + pip install prisma==0.11.0 + pip install "detect_secrets==1.5.0" + pip install "httpx==0.24.1" + pip install "respx==0.22.0" + pip install fastapi + pip install "gunicorn==21.2.0" + pip install "anyio==4.2.0" + pip install "aiodynamo==23.10.1" + pip install "asyncio==3.4.3" + pip install "apscheduler==3.10.4" + pip install "PyGithub==1.59.1" + pip install argon2-cffi + pip install "pytest-mock==3.12.0" + pip install python-multipart + pip install google-cloud-aiplatform + pip install prometheus-client==0.20.0 + pip install "pydantic==2.10.2" + pip install "diskcache==5.6.1" + pip install "Pillow==10.3.0" + pip install "jsonschema==4.22.0" + pip install "pytest-postgresql==7.0.1" + pip install "fakeredis==2.28.1" + pip install "pytest-xdist==3.6.1" + - setup_litellm_enterprise_pip + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Run prisma ./docker/entrypoint.sh + command: | + set +e + chmod +x docker/entrypoint.sh + ./docker/entrypoint.sh + set -e + - run: + name: Run proxy unit tests (part 2 - remaining tests) + command: | + pwd + ls + python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part2.xml --durations=10 -n 8 --timeout=300 -vv --log-cli-level=INFO + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_proxy_unit_tests_part2_coverage.xml + mv .coverage litellm_proxy_unit_tests_part2_coverage + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_proxy_unit_tests_part2_coverage.xml + - litellm_proxy_unit_tests_part2_coverage + litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword + docker: + - image: cimg/python:3.13.1 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + pip install wheel + pip install --upgrade pip wheel setuptools + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "respx==0.22.0" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install "pytest-cov==5.0.0" + # Run pytest and generate JUnit XML report + - setup_litellm_enterprise_pip + - run: + name: Run tests + command: | + pwd + ls + python -m pytest tests/local_testing/ -vv -k "assistants" --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_assistants_api_coverage.xml + mv .coverage litellm_assistants_api_coverage + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_assistants_api_coverage.xml + - litellm_assistants_api_coverage + llm_translation_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + pip install "pytest-xdist==3.6.1" + pip install "pytest-timeout==2.2.0" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + # Add --timeout to kill hanging tests after 120s (2 min) + # Add --durations=20 to show 20 slowest tests for debugging + python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml llm_translation_coverage.xml + mv .coverage llm_translation_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - llm_translation_coverage.xml + - llm_translation_coverage + mcp_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + pip install "pydantic==2.11.0" + pip install "mcp==1.25.0" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/mcp_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml mcp_coverage.xml + mv .coverage mcp_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - mcp_coverage.xml + - mcp_coverage + guardrails_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + pip install "pydantic==2.10.2" + pip install "boto3==1.36.0" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/guardrails_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml guardrails_coverage.xml + mv .coverage guardrails_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - guardrails_coverage.xml + - guardrails_coverage + + google_generate_content_endpoint_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + pip install "pydantic==2.10.2" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml google_generate_content_endpoint_coverage.xml + mv .coverage google_generate_content_endpoint_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - google_generate_content_endpoint_coverage.xml + - google_generate_content_endpoint_coverage + + llm_responses_api_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/llm_responses_api_testing --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml llm_responses_api_coverage.xml + mv .coverage llm_responses_api_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - llm_responses_api_coverage.xml + - llm_responses_api_coverage + ocr_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml ocr_coverage.xml + mv .coverage ocr_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - ocr_coverage.xml + - ocr_coverage + search_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml search_coverage.xml + mv .coverage search_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - search_coverage.xml + - search_coverage + # Split litellm_mapped_tests into 3 parallel jobs for 3x faster execution + litellm_mapped_tests_proxy: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: xlarge + steps: + - setup_litellm_test_deps + - run: + name: Run proxy tests + command: | + prisma generate + python -m pytest tests/test_litellm/proxy --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_proxy_tests_coverage.xml + mv .coverage litellm_proxy_tests_coverage + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_proxy_tests_coverage.xml + - litellm_proxy_tests_coverage + litellm_mapped_tests_llms: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: xlarge + steps: + - setup_litellm_test_deps + - run: + name: Run LLM provider tests + command: | + python -m pytest tests/test_litellm/llms --cov=litellm --cov-report=xml --junitxml=test-results/junit-llms.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_llms_tests_coverage.xml + mv .coverage litellm_llms_tests_coverage + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_llms_tests_coverage.xml + - litellm_llms_tests_coverage + litellm_mapped_tests_core: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: xlarge + steps: + - setup_litellm_test_deps + - run: + name: Run core tests + command: | + python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_core_tests_coverage.xml + mv .coverage litellm_core_tests_coverage + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_core_tests_coverage.xml + - litellm_core_tests_coverage + litellm_mapped_tests_litellm_core_utils: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: xlarge + steps: + - setup_litellm_test_deps + - run: + name: Run litellm_core_utils tests + command: | + python -m pytest tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-litellm-core-utils.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_core_utils_tests_coverage.xml + mv .coverage litellm_core_utils_tests_coverage + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_core_utils_tests_coverage.xml + - litellm_core_utils_tests_coverage + litellm_mapped_tests_integrations: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: xlarge + steps: + - setup_litellm_test_deps + - run: + name: Run integrations tests + command: | + python -m pytest tests/test_litellm/integrations --cov=litellm --cov-report=xml --junitxml=test-results/junit-integrations.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_integrations_tests_coverage.xml + mv .coverage litellm_integrations_tests_coverage + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_integrations_tests_coverage.xml + - litellm_integrations_tests_coverage + litellm_mapped_enterprise_tests: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest-mock==3.12.0" + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + pip install "hypercorn==0.17.3" + pip install "pydantic==2.11.0" + pip install "mcp==1.25.0" + pip install "requests-mock>=1.12.1" + pip install "responses==0.25.7" + pip install "pytest-xdist==3.6.1" + pip install "semantic_router==0.1.10" + pip install "fastapi-offline==1.7.3" + - setup_litellm_enterprise_pip + - run: + name: Run enterprise tests + command: | + pwd + ls + prisma generate + python -m pytest -vv tests/enterprise --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit-enterprise.xml --durations=10 -n 8 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_mapped_tests_coverage.xml + mv .coverage litellm_mapped_tests_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_mapped_tests_coverage.xml + - litellm_mapped_tests_coverage + batches_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "respx==0.22.0" + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install "pytest-cov==5.0.0" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml batches_coverage.xml + mv .coverage batches_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - batches_coverage.xml + - batches_coverage + litellm_utils_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + pip install numpydoc + python -m pip install -r requirements.txt + pip install "respx==0.22.0" + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install "pytest-cov==5.0.0" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install pytest-mock + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_utils_coverage.xml + mv .coverage litellm_utils_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_utils_coverage.xml + - litellm_utils_coverage + + pass_through_unit_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml pass_through_unit_tests_coverage.xml + mv .coverage pass_through_unit_tests_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - pass_through_unit_tests_coverage.xml + - pass_through_unit_tests_coverage + image_gen_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + pip install "pytest-xdist==3.6.1" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/image_gen_tests -n 4 --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml image_gen_coverage.xml + mv .coverage image_gen_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - image_gen_coverage.xml + - image_gen_coverage + logging_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install pytest-mock + pip install "respx==0.22.0" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install "mlflow==2.17.2" + pip install "anthropic==0.52.0" + pip install "blockbuster==1.5.24" + pip install "pytest-xdist==3.6.1" + # Run pytest and generate JUnit XML report + - setup_litellm_enterprise_pip + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/logging_callback_tests --cov=litellm -n 4 --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml logging_coverage.xml + mv .coverage logging_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - logging_coverage.xml + - logging_coverage + audio_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml audio_coverage.xml + mv .coverage audio_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - audio_coverage.xml + - audio_coverage + installing_litellm_on_python: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + pip install python-dotenv + pip install pytest + pip install tiktoken + pip install aiohttp + pip install openai + pip install click + pip install "boto3==1.36.0" + pip install jinja2 + pip install "tokenizers==0.20.0" + pip install "uvloop==0.21.0" + pip install "fastuuid==0.12.0" + pip install jsonschema + - setup_litellm_enterprise_pip + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/local_testing/test_basic_python_version.py + + installing_litellm_on_python_3_13: + docker: + - image: cimg/python:3.13.1 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install wheel setuptools + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install "pytest-cov==5.0.0" + pip install "tomli==2.2.1" + pip install "mcp==1.25.0" + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/local_testing/test_basic_python_version.py + helm_chart_testing: + machine: + image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker + resource_class: medium + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + # Install Helm + - run: + name: Install Helm + command: | + curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + + # Install kind + - run: + name: Install Kind + command: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind + + # Install kubectl + - run: + name: Install kubectl + command: | + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + sudo mv kubectl /usr/local/bin/ + + # Create kind cluster + - run: + name: Create Kind Cluster + command: | + kind create cluster --name litellm-test + + - run: + name: Build Docker image for helm tests + command: | + IMAGE_TAG=${CIRCLE_SHA1:-ci} + docker build -t litellm-ci:${IMAGE_TAG} -f docker/Dockerfile.database . + + - run: + name: Load Docker image into Kind + command: | + IMAGE_TAG=${CIRCLE_SHA1:-ci} + kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test + + # Run helm lint + - run: + name: Run helm lint + command: | + helm lint ./deploy/charts/litellm-helm + + # Run helm tests + - run: + name: Run helm tests + command: | + IMAGE_TAG=${CIRCLE_SHA1:-ci} + helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \ + --set image.repository=litellm-ci \ + --set image.tag=${IMAGE_TAG} \ + --set image.pullPolicy=Never + # Wait for pod to be ready + echo "Waiting 30 seconds for pod to be ready..." + sleep 30 + + # Print pod logs before running tests + echo "Printing pod logs..." + kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}") + + # Run the helm tests + helm test litellm --logs + helm test litellm --logs + + # Cleanup + - run: + name: Cleanup + command: | + kind delete cluster --name litellm-test + when: always # This ensures cleanup runs even if previous steps fail + + check_code_and_doc_quality: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project/litellm + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + pip install ruff + pip install pylint + pip install pyright + pip install beautifulsoup4 + pip install . + curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + - run: python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) + - run: ruff check ./litellm + # - run: python ./tests/documentation_tests/test_general_setting_keys.py + - run: python ./tests/code_coverage_tests/check_licenses.py + - run: python ./tests/code_coverage_tests/check_provider_folders_documented.py + - run: python ./tests/code_coverage_tests/router_code_coverage.py + - run: python ./tests/code_coverage_tests/test_chat_completion_imports.py + - run: python ./tests/code_coverage_tests/info_log_check.py + - run: python ./tests/code_coverage_tests/test_ban_set_verbose.py + - run: python ./tests/code_coverage_tests/code_qa_check_tests.py + - run: python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py + - run: python ./tests/code_coverage_tests/test_proxy_types_import.py + - run: python ./tests/code_coverage_tests/callback_manager_test.py + - run: python ./tests/code_coverage_tests/recursive_detector.py + - run: python ./tests/code_coverage_tests/test_router_strategy_async.py + - run: python ./tests/code_coverage_tests/litellm_logging_code_coverage.py + - run: python ./tests/documentation_tests/test_env_keys.py + - run: python ./tests/documentation_tests/test_router_settings.py + - run: python ./tests/documentation_tests/test_api_docs.py + - run: python ./tests/code_coverage_tests/ensure_async_clients_test.py + - run: python ./tests/code_coverage_tests/enforce_llms_folder_style.py + - run: python ./tests/documentation_tests/test_circular_imports.py + - run: python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py + - run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py + - run: python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py + - run: python ./tests/code_coverage_tests/check_fastuuid_usage.py + - run: python ./tests/code_coverage_tests/memory_test.py + - run: helm lint ./deploy/charts/litellm-helm + + db_migration_disable_update_check: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Python 3.9 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.9 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install aiohttp + pip install apscheduler + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + gunzip -c litellm-docker-database.tar.gz | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ + -e DISABLE_SCHEMA_UPDATE="True" \ + -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ + -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ + -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ + --name my-app \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 + - run: + name: Install curl and dockerize + command: | + sudo apt-get update + sudo apt-get install -y curl + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + + - run: + name: Wait for container to be ready + command: dockerize -wait http://localhost:4000 -timeout 1m + - run: + name: Check container logs for expected message + command: | + echo "=== Printing Full Container Startup Logs ===" + LOG_OUTPUT="$(docker logs my-app 2>&1)" + printf '%s\n' "$LOG_OUTPUT" + echo "=== End of Full Container Startup Logs ===" + + if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then + echo "Expected message found in logs. Test passed." + else + echo "Expected message not found in logs. Test failed." + exit 1 + fi + - run: + name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) + command: | + python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + no_output_timeout: 120m + + build_and_test: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + - run: + name: Install Python 3.9 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.9 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install aiohttp + python -m pip install --upgrade pip + python -m pip install -r .circleci/requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-mock==3.12.0" + pip install "pytest-asyncio==0.21.1" + pip install "mypy==1.18.2" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install pyarrow + pip install "boto3==1.36.0" + pip install "aioboto3==13.4.0" + pip install langchain + pip install "langfuse>=2.0.0" + pip install "logfire==0.29.0" + pip install numpydoc + pip install prisma + pip install fastapi + pip install jsonschema + pip install "httpx==0.24.1" + pip install "gunicorn==21.2.0" + pip install "anyio==3.7.1" + pip install "aiodynamo==23.10.1" + pip install "asyncio==3.4.3" + pip install "PyGithub==1.59.1" + pip install "openai==1.100.1" + pip install "litellm[proxy]" + pip install "pytest-xdist==3.6.1" + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - run: + name: Build Docker image + command: docker build -t my-app:latest -f ./docker/Dockerfile.database . + - run: + name: Run Docker container + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e USE_PRISMA_MIGRATE=True \ + -e AZURE_API_KEY=$AZURE_API_KEY \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ + -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ + -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e GROQ_API_KEY=$GROQ_API_KEY \ + -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -e COHERE_API_KEY=$COHERE_API_KEY \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e AWS_REGION_NAME=$AWS_REGION_NAME \ + -e AUTO_INFER_REGION=True \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ + -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ + -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ + -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \ + my-app:latest \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Install curl + command: | + sudo apt-get update + sudo apt-get install -y curl + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests + no_output_timeout: 120m + + # Store test results + - store_test_results: + path: test-results + e2e_openai_endpoints: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + - run: + name: Install Python 3.10 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.10 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install aiohttp + python -m pip install --upgrade pip + python -m pip install -r .circleci/requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-mock==3.12.0" + pip install "pytest-asyncio==0.21.1" + pip install "mypy==1.18.2" + pip install "jsonlines==4.0.0" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install pyarrow + pip install "boto3==1.36.0" + pip install "aioboto3==13.4.0" + pip install langchain + pip install "langchain_mcp_adapters==0.0.5" + pip install "langfuse>=2.0.0" + pip install "logfire==0.29.0" + pip install numpydoc + pip install prisma + pip install fastapi + pip install jsonschema + pip install "httpx==0.24.1" + pip install "gunicorn==21.2.0" + pip install "anyio==3.7.1" + pip install "aiodynamo==23.10.1" + pip install "asyncio==3.4.3" + pip install "PyGithub==1.59.1" + pip install "openai==1.100.1" + # Run pytest and generate JUnit XML report + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + gunzip -c litellm-docker-database.tar.gz | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e AZURE_API_KEY=$AZURE_API_KEY \ + -e AZURE_API_BASE=$AZURE_API_BASE \ + -e AZURE_API_VERSION="2024-05-01-preview" \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ + -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ + -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e GROQ_API_KEY=$GROQ_API_KEY \ + -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -e COHERE_API_KEY=$COHERE_API_KEY \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e AWS_REGION_NAME=$AWS_REGION_NAME \ + -e AUTO_INFER_REGION=True \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ + -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ + -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ + -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Install curl + command: | + sudo apt-get update + sudo apt-get install -y curl + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + + # Store test results + - store_test_results: + path: test-results + proxy_logging_guardrails_model_info_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + - run: + name: Install Python 3.9 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.9 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install aiohttp + python -m pip install --upgrade pip + python -m pip install -r .circleci/requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-mock==3.12.0" + pip install "pytest-asyncio==0.21.1" + pip install "mypy==1.18.2" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install pyarrow + pip install "boto3==1.36.0" + pip install "aioboto3==13.4.0" + pip install langchain + pip install "langfuse>=2.0.0" + pip install "logfire==0.29.0" + pip install numpydoc + pip install prisma + pip install fastapi + pip install jsonschema + pip install "httpx==0.24.1" + pip install "gunicorn==21.2.0" + pip install "anyio==3.7.1" + pip install "aiodynamo==23.10.1" + pip install "asyncio==3.4.3" + pip install "PyGithub==1.59.1" + pip install "openai==1.100.1" + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + gunzip -c litellm-docker-database.tar.gz | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container + # intentionally give bad redis credentials here + # the OTEL test - should get this as a trace + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e OTEL_EXPORTER="in_memory" \ + -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ + -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ + -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + -e AWS_REGION_NAME=$AWS_REGION_NAME \ + -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ + -e COHERE_API_KEY=$COHERE_API_KEY \ + -e GCS_FLUSH_INTERVAL="1" \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \ + -v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Install curl and dockerize + command: | + sudo apt-get update + sudo apt-get install -y curl + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: + 120m + # Clean up first container + - run: + name: Stop and remove first container + command: | + docker stop my-app + docker rm my-app + + # Second Docker Container Run with Different Config + # NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license + - run: + name: Run Second Docker container + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e LITELLM_LICENSE="bad-license" \ + --add-host host.docker.internal:host-gateway \ + --name my-app-3 \ + -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug + + - run: + name: Start outputting logs for second container + command: docker logs -f my-app-2 + background: true + + - run: + name: Wait for second app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + + - run: + name: Run second round of tests + command: | + python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + no_output_timeout: 120m + + # Store test results + - store_test_results: + path: test-results + proxy_spend_accuracy_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + - run: + name: Install Python 3.9 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.9 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install aiohttp + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + gunzip -c litellm-docker-database.tar.gz | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container + # intentionally give bad redis credentials here + # the OTEL test - should get this as a trace + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + -e AWS_REGION_NAME=$AWS_REGION_NAME \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Install curl + command: | + sudo apt-get update + sudo apt-get install -y curl + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: + 120m + # Clean up first container + - run: + name: Stop and remove first container + command: | + docker stop my-app + docker rm my-app + + proxy_multi_instance_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + - run: + name: Install Python 3.9 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.9 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install aiohttp + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-mock==3.12.0" + pip install "pytest-asyncio==0.21.1" + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + gunzip -c litellm-docker-database.tar.gz | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container 1 + # intentionally give bad redis credentials here + # the OTEL test - should get this as a trace + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Run Docker container 2 + command: | + docker run -d \ + -p 4001:4001 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + --add-host host.docker.internal:host-gateway \ + --name my-app-2 \ + -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4001 \ + --detailed_debug + - run: + name: Install curl and dockerize + command: | + sudo apt-get update + sudo apt-get install -y curl + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for instance 1 to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Wait for instance 2 to be ready + command: dockerize -wait http://localhost:4001 -timeout 5m + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: + 120m + # Clean up first container + # Store test results + - store_test_results: + path: test-results + + proxy_store_model_in_db_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + sudo systemctl restart docker + - run: + name: Install Python 3.9 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.9 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install aiohttp + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-mock==3.12.0" + pip install "pytest-asyncio==0.21.1" + pip install "assemblyai==0.37.0" + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + gunzip -c litellm-docker-database.tar.gz | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container + # intentionally give bad redis credentials here + # the OTEL test - should get this as a trace + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e STORE_MODEL_IN_DB="True" \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Install curl and dockerize + command: | + sudo apt-get update + sudo apt-get install -y curl + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Stop and remove containers + command: | + docker stop my-app || true + docker rm my-app || true + docker stop postgres-db || true + docker rm postgres-db || true + when: always + - store_test_results: + path: test-results + + proxy_build_from_pip_tests: + # Change from docker to machine executor + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + # Remove Docker CLI installation since it's already available in machine executor + - run: + name: Install Python 3.13 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.13 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install aiohttp + python -m pip install --upgrade pip + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-mock==3.12.0" + pip install "pytest-asyncio==0.21.1" + pip install "mypy==1.18.2" + pip install apscheduler + - run: + name: Build Docker image + command: | + docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip . + - run: + name: Run Docker container + # intentionally give bad redis credentials here + # the OTEL test - should get this as a trace + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e OTEL_EXPORTER="in_memory" \ + -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ + -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ + -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e AWS_REGION_NAME=$AWS_REGION_NAME \ + -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ + -e COHERE_API_KEY=$COHERE_API_KEY \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + -e GCS_FLUSH_INTERVAL="1" \ + --name my-app \ + -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \ + my-app:latest \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Install curl and dockerize + command: | + sudo apt-get update + sudo apt-get install -y curl + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Run tests + command: | + python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + no_output_timeout: + 120m + # Clean up first container + - run: + name: Stop and remove first container + command: | + docker stop my-app + docker rm my-app + proxy_pass_through_endpoint_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + - run: + name: Install Python 3.10 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.10 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install "google-cloud-aiplatform==1.43.0" + pip install aiohttp + pip install "openai==1.100.1" + pip install "assemblyai==0.37.0" + python -m pip install --upgrade pip + pip install "pydantic==2.10.2" + pip install "pytest==7.3.1" + pip install "pytest-mock==3.12.0" + pip install "pytest-asyncio==0.21.1" + pip install "boto3==1.36.0" + pip install "mypy==1.18.2" + pip install pyarrow + pip install numpydoc + pip install prisma + pip install fastapi + pip install jsonschema + pip install "httpx==0.27.0" + pip install "anyio==3.7.1" + pip install "asyncio==3.4.3" + pip install "PyGithub==1.59.1" + pip install "google-cloud-aiplatform==1.59.0" + pip install "anthropic==0.52.0" + pip install "langchain_mcp_adapters==0.0.5" + pip install "langchain_openai==0.2.1" + pip install "langgraph==0.3.18" + pip install "fastuuid==0.13.5" + pip install -r requirements.txt + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + gunzip -c litellm-docker-database.tar.gz | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e GEMINI_API_KEY=$GEMINI_API_KEY \ + -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \ + -e AZURE_API_KEY=$AZURE_API_KEY \ + -e AZURE_API_BASE=$AZURE_API_BASE \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \ + -v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + # Add Ruby installation and testing before the existing Node.js and Python tests + - run: + name: Install Ruby and Bundler + command: | + # Import GPG keys first + gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB || { + curl -sSL https://rvm.io/mpapis.asc | gpg --import - + curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - + } + + # Install Ruby version manager (RVM) + curl -sSL https://get.rvm.io | bash -s stable + + # Source RVM from the correct location + source $HOME/.rvm/scripts/rvm + + # Install Ruby 3.2.2 + rvm install 3.2.2 + rvm use 3.2.2 --default + + # Install latest Bundler + gem install bundler + + - run: + name: Run Ruby tests + command: | + source $HOME/.rvm/scripts/rvm + cd tests/pass_through_tests/ruby_passthrough_tests + bundle install + bundle exec rspec + no_output_timeout: 30m + # New steps to run Node.js test + - run: + name: Install Node.js + command: | + export DEBIAN_FRONTEND=noninteractive + curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash - + sudo apt-get update + sudo apt-get install -y nodejs + node --version + npm --version + + - run: + name: Install Node.js dependencies + command: | + npm install @google-cloud/vertexai + npm install @google/generative-ai + npm install --save-dev jest + + - run: + name: Run Vertex AI, Google AI Studio Node.js tests + command: | + npx jest tests/pass_through_tests --verbose + no_output_timeout: 30m + - run: + name: Run tests + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + pwd + ls + python -m pytest -vv tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + + # Store test results + - store_test_results: + path: test-results + + proxy_e2e_anthropic_messages_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + - run: + name: Install Python 3.10 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.10 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install "boto3==1.36.0" + pip install "httpx==0.27.0" + pip install "claude-agent-sdk" + pip install -r requirements.txt + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + gunzip -c litellm-docker-database.tar.gz | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container with test config + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e AWS_REGION_NAME="us-east-1" \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Run Claude Agent SDK E2E Tests + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + export LITELLM_PROXY_URL="http://localhost:4000" + export LITELLM_API_KEY="sk-1234" + pwd + ls + python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + + # Store test results + - store_test_results: + path: test-results upload-coverage: docker: @@ -4014,361 +4014,361 @@ workflows: only: - main - /litellm_.*/ - # - local_testing_part1: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - local_testing_part2: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - langfuse_logging_unit_tests: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - caching_unit_tests: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - litellm_proxy_unit_testing_key_generation: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - litellm_proxy_unit_testing_part1: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - litellm_proxy_unit_testing_part2: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - litellm_security_tests: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - litellm_assistants_api_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - litellm_router_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - litellm_router_unit_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - check_code_and_doc_quality: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - ui_build: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - ui_unit_tests: - # requires: - # - ui_build - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - auth_ui_unit_tests: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - build_docker_database_image: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - e2e_ui_testing: - # context: e2e_ui_tests - # requires: - # - ui_build - # - build_docker_database_image - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - build_and_test: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - e2e_openai_endpoints: - # requires: - # - build_docker_database_image - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - proxy_logging_guardrails_model_info_tests: - # requires: - # - build_docker_database_image - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - proxy_spend_accuracy_tests: - # requires: - # - build_docker_database_image - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - proxy_multi_instance_tests: - # requires: - # - build_docker_database_image - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - proxy_store_model_in_db_tests: - # requires: - # - build_docker_database_image - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - proxy_build_from_pip_tests: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - proxy_pass_through_endpoint_tests: - # requires: - # - build_docker_database_image - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - proxy_e2e_anthropic_messages_tests: - # requires: - # - build_docker_database_image - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - llm_translation_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - mcp_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - guardrails_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - google_generate_content_endpoint_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - llm_responses_api_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - ocr_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - search_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - litellm_mapped_enterprise_tests: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - litellm_mapped_tests_proxy: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - litellm_mapped_tests_llms: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - litellm_mapped_tests_core: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - litellm_mapped_tests_integrations: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - litellm_mapped_tests_litellm_core_utils: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - batches_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - litellm_utils_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - pass_through_unit_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - image_gen_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - logging_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - audio_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - upload-coverage: - # requires: - # - llm_translation_testing - # - mcp_testing - # - google_generate_content_endpoint_testing - # - guardrails_testing - # - llm_responses_api_testing - # - ocr_testing - # - search_testing - # - litellm_mapped_tests_proxy - # - litellm_mapped_tests_llms - # - litellm_mapped_tests_core - # - litellm_mapped_tests_integrations - # - litellm_mapped_tests_litellm_core_utils - # - litellm_mapped_enterprise_tests - # - batches_testing - # - litellm_utils_testing - # - pass_through_unit_testing - # - image_gen_testing - # - logging_testing - # - audio_testing - # - litellm_router_testing - # - litellm_router_unit_testing - # - caching_unit_tests - # - litellm_proxy_unit_testing_key_generation - # - litellm_proxy_unit_testing_part1 - # - litellm_proxy_unit_testing_part2 - # - litellm_security_tests - # - langfuse_logging_unit_tests - # - local_testing_part1 - # - local_testing_part2 - # - litellm_assistants_api_testing - # - auth_ui_unit_tests - # - db_migration_disable_update_check: - # requires: - # - build_docker_database_image - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - installing_litellm_on_python: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - installing_litellm_on_python_3_13: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - helm_chart_testing: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ - # - test_bad_database_url: - # filters: - # branches: - # only: - # - main - # - /litellm_.*/ + - local_testing_part1: + filters: + branches: + only: + - main + - /litellm_.*/ + - local_testing_part2: + filters: + branches: + only: + - main + - /litellm_.*/ + - langfuse_logging_unit_tests: + filters: + branches: + only: + - main + - /litellm_.*/ + - caching_unit_tests: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_proxy_unit_testing_key_generation: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_proxy_unit_testing_part1: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_proxy_unit_testing_part2: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_security_tests: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_assistants_api_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_router_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_router_unit_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - check_code_and_doc_quality: + filters: + branches: + only: + - main + - /litellm_.*/ + - ui_build: + filters: + branches: + only: + - main + - /litellm_.*/ + - ui_unit_tests: + requires: + - ui_build + filters: + branches: + only: + - main + - /litellm_.*/ + - auth_ui_unit_tests: + filters: + branches: + only: + - main + - /litellm_.*/ + - build_docker_database_image: + filters: + branches: + only: + - main + - /litellm_.*/ + - e2e_ui_testing: + context: e2e_ui_tests + requires: + - ui_build + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ + - build_and_test: + filters: + branches: + only: + - main + - /litellm_.*/ + - e2e_openai_endpoints: + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ + - proxy_logging_guardrails_model_info_tests: + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ + - proxy_spend_accuracy_tests: + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ + - proxy_multi_instance_tests: + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ + - proxy_store_model_in_db_tests: + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ + - proxy_build_from_pip_tests: + filters: + branches: + only: + - main + - /litellm_.*/ + - proxy_pass_through_endpoint_tests: + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ + - proxy_e2e_anthropic_messages_tests: + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ + - llm_translation_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - mcp_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - guardrails_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - google_generate_content_endpoint_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - llm_responses_api_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - ocr_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - search_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_mapped_enterprise_tests: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_mapped_tests_proxy: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_mapped_tests_llms: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_mapped_tests_core: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_mapped_tests_integrations: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_mapped_tests_litellm_core_utils: + filters: + branches: + only: + - main + - /litellm_.*/ + - batches_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_utils_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - pass_through_unit_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - image_gen_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - logging_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - audio_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - upload-coverage: + requires: + - llm_translation_testing + - mcp_testing + - google_generate_content_endpoint_testing + - guardrails_testing + - llm_responses_api_testing + - ocr_testing + - search_testing + - litellm_mapped_tests_proxy + - litellm_mapped_tests_llms + - litellm_mapped_tests_core + - litellm_mapped_tests_integrations + - litellm_mapped_tests_litellm_core_utils + - litellm_mapped_enterprise_tests + - batches_testing + - litellm_utils_testing + - pass_through_unit_testing + - image_gen_testing + - logging_testing + - audio_testing + - litellm_router_testing + - litellm_router_unit_testing + - caching_unit_tests + - litellm_proxy_unit_testing_key_generation + - litellm_proxy_unit_testing_part1 + - litellm_proxy_unit_testing_part2 + - litellm_security_tests + - langfuse_logging_unit_tests + - local_testing_part1 + - local_testing_part2 + - litellm_assistants_api_testing + - auth_ui_unit_tests + - db_migration_disable_update_check: + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ + - installing_litellm_on_python: + filters: + branches: + only: + - main + - /litellm_.*/ + - installing_litellm_on_python_3_13: + filters: + branches: + only: + - main + - /litellm_.*/ + - helm_chart_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - test_bad_database_url: + filters: + branches: + only: + - main + - /litellm_.*/ - publish_proxy_extras: filters: branches: @@ -4378,4 +4378,49 @@ workflows: - publish_to_pypi: requires: - mypy_linting + - local_testing_part1 + - local_testing_part2 + - build_and_test + - e2e_openai_endpoints + - test_bad_database_url + - llm_translation_testing + - mcp_testing + - google_generate_content_endpoint_testing + - llm_responses_api_testing + - ocr_testing + - search_testing + - litellm_mapped_tests_proxy + - litellm_mapped_tests_llms + - litellm_mapped_tests_core + - litellm_mapped_tests_integrations + - litellm_mapped_tests_litellm_core_utils + - litellm_mapped_enterprise_tests + - batches_testing + - litellm_utils_testing + - pass_through_unit_testing + - image_gen_testing + - logging_testing + - audio_testing + - litellm_router_testing + - litellm_router_unit_testing + - caching_unit_tests + - langfuse_logging_unit_tests + - litellm_assistants_api_testing + - auth_ui_unit_tests + - db_migration_disable_update_check + - e2e_ui_testing + - litellm_proxy_unit_testing_key_generation + - litellm_proxy_unit_testing_part1 + - litellm_proxy_unit_testing_part2 + - litellm_security_tests + - installing_litellm_on_python + - installing_litellm_on_python_3_13 + - proxy_logging_guardrails_model_info_tests + - proxy_spend_accuracy_tests + - proxy_multi_instance_tests + - proxy_store_model_in_db_tests + - proxy_build_from_pip_tests + - proxy_pass_through_endpoint_tests + - check_code_and_doc_quality - publish_proxy_extras + - guardrails_testing