litellm/.circleci/config.yml

version: 2.1
orbs:
  codecov: codecov/codecov@4.0.1
  node: circleci/node@5.1.0 # Add this line to declare the node orb
  win: circleci/windows@5.0 # Add Windows orb

commands:
  setup_google_dns:
    steps:
      - run:
          name: "Configure Google DNS"
          command: |
            # Backup original resolv.conf
            sudo cp /etc/resolv.conf /etc/resolv.conf.backup
            # Add both local and Google DNS servers
            echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf
            echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf
            echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf
  setup_litellm_enterprise_pip:
    steps:
      - run:
          name: "Install local version of litellm-enterprise"
          command: |
            pip install --force-reinstall --no-deps -e enterprise/
  setup_litellm_test_deps:
    steps:
      - checkout
      - setup_google_dns
      - restore_cache:
          keys:
            - v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }}
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            # Use uv for the heavy requirements.txt (10-100x faster than pip)
            uv pip install --system -r requirements.txt
            # Use pip for test deps (small set, avoids uv strict-resolution
            # conflicts with transitive dep pins like openai<2 and pydantic>=2.11.5)
            pip install "pytest-mock==3.12.0" "pytest==7.3.1" "pytest-retry==1.6.3" \
              "pytest-asyncio==0.21.1" "respx==0.22.0" "hypercorn==0.17.3" \
              "pydantic==2.12.5" "mcp==1.26.0" "requests-mock>=1.12.1" \
              "responses==0.25.7" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" \
              "pytest-cov==5.0.0" "semantic_router==0.1.10" "fastapi-offline==1.7.3" \
              "a2a" "parameterized>=0.9.0"
      - setup_litellm_enterprise_pip
      - save_cache:
          paths:
            - ~/.local/lib
            - ~/.local/bin
            - ~/.cache/uv
          key: v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }}

jobs:
  # Add Windows testing job
  using_litellm_on_windows:
    executor:
      name: win/default
      shell: powershell.exe
    working_directory: ~/project
    steps:
      - checkout
      - run:
          name: Install Python
          command: |
            choco install python --version=3.11.0 -y --no-progress --force
            refreshenv
            python --version
          environment:
            CHOCOLATEY_CONFIRM_ALL: "true"
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip
            pip install pytest
            pip install .
      - run:
          name: Run Windows-specific test
          command: |
            python -m pytest tests/windows_tests/test_litellm_on_windows.py -v

  mypy_linting:
    docker:
      - image: cimg/python:3.12
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project
    resource_class: medium

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip uninstall fastuuid -y
            pip install "mypy==1.18.2"
      - run:
          name: MyPy Type Checking
          command: |
            cd litellm
            # Use the same approach as GitHub Actions, explicitly exclude fastuuid to avoid segfaults
            python -m mypy .
            cd ..
          no_output_timeout: 10m

  semgrep:
    docker:
      - image: cimg/python:3.12
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project
    resource_class: medium
    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Semgrep
          command: pip install semgrep
      - run:
          name: Run Semgrep (custom rules only)
          command: semgrep scan --config .semgrep/rules . --error

  local_testing_part1:
    docker:
      - image: cimg/python:3.12
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project
    parallelism: 4
    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Show git commit hash
          command: |
            echo "Git commit hash: $CIRCLE_SHA1"

      - restore_cache:
          keys:
            - v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip
            python -m pip install -r .circleci/requirements.txt
            pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \
              "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \
              "boto3==1.42.80" langchain lunary==0.2.5 \
              "azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \
              traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \
              "detect_secrets==1.5.0" "respx==0.22.0" fastapi \
              "gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \
              "apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \
              python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \
              "diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \
              "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1"
            pip install semantic_router --no-deps
            pip install aurelio_sdk --no-deps
            pip uninstall posthog -y
      - setup_litellm_enterprise_pip
      - save_cache:
          paths:
            - ./venv
          key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
      - run:
          name: Run prisma ./docker/entrypoint.sh
          command: |
            set +e
            chmod +x docker/entrypoint.sh
            ./docker/entrypoint.sh
            set -e
      - run:
          name: Black Formatting
          command: |
            cd litellm
            python -m pip install black
            python -m black .
            cd ..

      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests (Part 1 - A-M)
          command: |
            mkdir test-results

            # Discover test files (A-M)
            TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py")

            echo "$TEST_FILES" | circleci tests run \
              --split-by=timings \
              --verbose \
              --command="xargs python -m pytest \
                -vv \
                --cov=litellm \
                --cov-report=xml \
                --junitxml=test-results/junit.xml \
                --durations=20 \
                -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
                -n 4 \
                --timeout=300 \
                --timeout_method=thread"
          no_output_timeout: 15m
      - run:
          name: Rename the coverage files
          command: |
            mv coverage.xml local_testing_part1_coverage.xml
            mv .coverage local_testing_part1_coverage

      # Store test results
      - store_test_results:
          path: test-results
      - persist_to_workspace:
          root: .
          paths:
            - local_testing_part1_coverage.xml
            - local_testing_part1_coverage
  local_testing_part2:
    docker:
      - image: cimg/python:3.12
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project
    parallelism: 4
    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Show git commit hash
          command: |
            echo "Git commit hash: $CIRCLE_SHA1"

      - restore_cache:
          keys:
            - v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip
            python -m pip install -r .circleci/requirements.txt
            pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \
              "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \
              "boto3==1.42.80" langchain lunary==0.2.5 \
              "azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \
              traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \
              "detect_secrets==1.5.0" "respx==0.22.0" fastapi \
              "gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \
              "apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \
              python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \
              "diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \
              "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1"
            pip install semantic_router --no-deps
            pip install aurelio_sdk --no-deps
            pip uninstall posthog -y
      - setup_litellm_enterprise_pip
      - save_cache:
          paths:
            - ./venv
          key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
      - run:
          name: Run prisma ./docker/entrypoint.sh
          command: |
            set +e
            chmod +x docker/entrypoint.sh
            ./docker/entrypoint.sh
            set -e
      - run:
          name: Black Formatting
          command: |
            cd litellm
            python -m pip install black
            python -m black .
            cd ..

      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests (Part 2 - N-Z)
          command: |
            mkdir test-results

            # Discover test files (N-Z)
            TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py")

            echo "$TEST_FILES" | circleci tests run \
              --split-by=timings \
              --verbose \
              --command="xargs python -m pytest \
                -vv \
                --cov=litellm \
                --cov-report=xml \
                --junitxml=test-results/junit.xml \
                --durations=20 \
                -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
                -n 4 \
                --timeout=300 \
                --timeout_method=thread"
          no_output_timeout: 15m
      - run:
          name: Rename the coverage files
          command: |
            mv coverage.xml local_testing_part2_coverage.xml
            mv .coverage local_testing_part2_coverage

      # Store test results
      - store_test_results:
          path: test-results
      - persist_to_workspace:
          root: .
          paths:
            - local_testing_part2_coverage.xml
            - local_testing_part2_coverage
  langfuse_logging_unit_tests:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project
    resource_class: medium

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Show git commit hash
          command: |
            echo "Git commit hash: $CIRCLE_SHA1"

      - restore_cache:
          keys:
            - v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip
            python -m pip install -r .circleci/requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-asyncio==0.21.1"
            pip install "pytest-cov==5.0.0"
            pip install "mypy==1.18.2"
            pip install "google-generativeai==0.3.2"
            pip install "google-cloud-aiplatform==1.133.0"
            pip install pyarrow
            pip install "boto3==1.42.80"
            pip install langchain
            pip install lunary==0.2.5
            pip install "azure-identity==1.25.3"
            pip install "langfuse==2.59.7"
            pip install "logfire==0.29.0"
            pip install numpydoc
            pip install traceloop-sdk==0.21.1
            pip install opentelemetry-api==1.28.0
            pip install opentelemetry-sdk==1.28.0
            pip install opentelemetry-exporter-otlp==1.28.0
            pip install openai==1.100.1
            pip install prisma==0.11.0
            pip install "detect_secrets==1.5.0"
            pip install "httpx==0.28.1"
            pip install "respx==0.22.0"
            pip install fastapi
            pip install "gunicorn==23.0.0"
            pip install "anyio==4.8.0"
            pip install "aiodynamo==23.10.1"
            pip install "asyncio==3.4.3"
            pip install "apscheduler==3.11.2"
            pip install "PyGithub==1.59.1"
            pip install argon2-cffi
            pip install "pytest-mock==3.12.0"
            pip install python-multipart
            pip install google-cloud-aiplatform
            pip install prometheus-client==0.20.0
            pip install "pydantic==2.12.5"
            pip install "diskcache==5.6.1"
            pip install "Pillow==12.1.1"
            pip install "jsonschema==4.23.0"
            pip install "websockets==15.0.1"
      - setup_litellm_enterprise_pip
      - save_cache:
          paths:
            - ./venv
          key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
      - run:
          name: Run prisma ./docker/entrypoint.sh
          command: |
            set +e
            chmod +x docker/entrypoint.sh
            ./docker/entrypoint.sh
            set -e

      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse"
          no_output_timeout: 15m
      # Store test results
      - store_test_results:
          path: test-results
  auth_ui_unit_tests:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-asyncio==0.21.1"
            pip install "pytest-xdist==3.6.1"
      - save_cache:
          paths:
            - ./venv
          key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
      - run:
          name: Run prisma ./docker/entrypoint.sh
          command: |
            set +e
            chmod +x docker/entrypoint.sh
            ./docker/entrypoint.sh
            set -e
      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2
          no_output_timeout: 15m

      # Store test results
      - store_test_results:
          path: test-results

  litellm_router_testing: # Runs all tests with the "router" keyword
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project
    resource_class: large
    parallelism: 4
    steps:
      - checkout
      - setup_google_dns
      - restore_cache:
          keys:
            - v1-router-testing-deps-{{ checksum "requirements.txt" }}
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "respx==0.22.0"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-asyncio==0.21.1"
            pip install "pytest-xdist==3.6.1"
            pip install "pytest-timeout==2.2.0"
            pip install semantic_router --no-deps
            pip install aurelio_sdk --no-deps
      - save_cache:
          paths:
            - /home/circleci/.pyenv
            - /home/circleci/.local
          key: v1-router-testing-deps-{{ checksum "requirements.txt" }}
      # Run pytest and generate JUnit XML report
      - setup_litellm_enterprise_pip
      - run:
          name: Run tests
          command: |
            pwd
            ls
            TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")

            echo "$TEST_FILES" | circleci tests run \
              --split-by=timings \
              --verbose \
              --command="xargs python -m pytest \
                -v \
                -k 'router' \
                -n 4 \
                --junitxml=test-results/junit.xml \
                --durations=5 \
                --timeout=300 --timeout_method=thread"
          no_output_timeout: 15m
      # Store test results
      - store_test_results:
          path: test-results

  litellm_router_unit_testing: # Runs all tests with the "router" keyword
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project
    resource_class: large

    steps:
      - checkout
      - setup_google_dns
      - restore_cache:
          keys:
            - v1-router-unit-deps-{{ checksum "requirements.txt" }}
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "respx==0.22.0"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-asyncio==0.21.1"
            pip install semantic_router --no-deps
            pip install aurelio_sdk --no-deps
            pip install "pytest-xdist==3.6.1"
      - save_cache:
          paths:
            - /home/circleci/.pyenv
            - /home/circleci/.local
          key: v1-router-unit-deps-{{ checksum "requirements.txt" }}
      # Run pytest and generate JUnit XML report
      - setup_litellm_enterprise_pip
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4
          no_output_timeout: 15m
      # Store test results
      - store_test_results:
          path: test-results
  litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
    docker:
      - image: cimg/python:3.13.1
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project
    resource_class: medium

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            pip install wheel setuptools
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "respx==0.22.0"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-asyncio==0.21.1"
      # Run pytest and generate JUnit XML report
      - setup_litellm_enterprise_pip
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5
          no_output_timeout: 15m
      # Store test results
      - store_test_results:
          path: test-results
  llm_translation_testing:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project
    resource_class: large

    steps:
      - checkout
      - setup_google_dns
      - restore_cache:
          keys:
            - v1-llm-translation-deps-{{ checksum "requirements.txt" }}
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-asyncio==0.21.1"
            pip install "respx==0.22.0"
            pip install "pytest-xdist==3.6.1"
            pip install "pytest-timeout==2.2.0"
      - save_cache:
          paths:
            - /home/circleci/.pyenv
            - /home/circleci/.local
          key: v1-llm-translation-deps-{{ checksum "requirements.txt" }}
      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests
          command: |
            pwd
            ls
            # Add --timeout to kill hanging tests after 120s (2 min)
            # Add --durations=20 to show 20 slowest tests for debugging
            # Subdirectories with dedicated jobs (maintain this list as new jobs are added)
            IGNORE_DIRS=(
              "tests/llm_translation/realtime"
            )
            IGNORE_ARGS=""
            for dir in "${IGNORE_DIRS[@]}"; do
              IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir"
            done
            python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5
          no_output_timeout: 15m

      # Store test results
      - store_test_results:
          path: test-results
  realtime_translation_testing:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets"
      # Run pytest and generate JUnit XML report
      - run:
          name: Run realtime tests
          command: |
            pwd
            ls
            # Add --timeout to kill hanging tests after 120s (2 min)
            # Add --durations=20 to show 20 slowest tests for debugging
            python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
          no_output_timeout: 15m
      - run:
          name: Rename the coverage files
          command: |
            mv coverage.xml realtime_translation_coverage.xml
            mv .coverage realtime_translation_coverage

      # Store test results
      - store_test_results:
          path: test-results
      - persist_to_workspace:
          root: .
          paths:
            - realtime_translation_coverage.xml
            - realtime_translation_coverage
  mcp_testing:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-cov==5.0.0"
            pip install "pytest-asyncio==0.21.1"
            pip install "respx==0.22.0"
            pip install "pydantic==2.12.5"
            pip install "mcp==1.26.0"
            pip install "pytest-xdist==3.6.1"
      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -vv tests/mcp_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
          no_output_timeout: 15m
      - run:
          name: Rename the coverage files
          command: |
            mv coverage.xml mcp_coverage.xml
            mv .coverage mcp_coverage

      # Store test results
      - store_test_results:
          path: test-results
      - persist_to_workspace:
          root: .
          paths:
            - mcp_coverage.xml
            - mcp_coverage
  agent_testing:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-cov==5.0.0"
            pip install "pytest-asyncio==0.21.1"
            pip install "respx==0.22.0"
            pip install "pydantic==2.12.5"
            pip install "a2a-sdk"
      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
          no_output_timeout: 15m
      - run:
          name: Rename the coverage files
          command: |
            mv coverage.xml agent_coverage.xml
            mv .coverage agent_coverage

      # Store test results
      - store_test_results:
          path: test-results
      - persist_to_workspace:
          root: .
          paths:
            - agent_coverage.xml
            - agent_coverage
  guardrails_testing:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-cov==5.0.0"
            pip install "pytest-asyncio==0.21.1"
            pip install "respx==0.22.0"
            pip install "pydantic==2.12.5"
            pip install "boto3==1.42.80"
            pip install "semantic_router==0.1.10" --no-deps
            pip install aurelio_sdk
            pip install "pytest-xdist==3.6.1"
            pip install "pytest-timeout==2.2.0"
      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests
          command: |
            pwd
            ls
            LITELLM_LOG=WARNING python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread
          no_output_timeout: 15m
      - run:
          name: Rename the coverage files
          command: |
            mv coverage.xml guardrails_coverage.xml
            mv .coverage guardrails_coverage

      # Store test results
      - store_test_results:
          path: test-results
      - persist_to_workspace:
          root: .
          paths:
            - guardrails_coverage.xml
            - guardrails_coverage

  google_generate_content_endpoint_testing:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-cov==5.0.0"
            pip install "pytest-asyncio==0.21.1"
            pip install "respx==0.22.0"
            pip install "pydantic==2.12.5"
      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5
          no_output_timeout: 15m
      - run:
          name: Rename the coverage files
          command: |
            mv coverage.xml google_generate_content_endpoint_coverage.xml
            mv .coverage google_generate_content_endpoint_coverage

      # Store test results
      - store_test_results:
          path: test-results
      - persist_to_workspace:
          root: .
          paths:
            - google_generate_content_endpoint_coverage.xml
            - google_generate_content_endpoint_coverage

  llm_responses_api_testing:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project
    resource_class: large

    steps:
      - checkout
      - setup_google_dns
      - restore_cache:
          keys:
            - v1-llm-responses-deps-{{ checksum "requirements.txt" }}
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-asyncio==0.21.1"
            pip install "respx==0.22.0"
            pip install "pytest-xdist==3.6.1"
      - save_cache:
          paths:
            - /home/circleci/.pyenv
            - /home/circleci/.local
          key: v1-llm-responses-deps-{{ checksum "requirements.txt" }}
      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8
          no_output_timeout: 15m

      # Store test results
      - store_test_results:
          path: test-results
  ocr_testing:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1"
      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
          no_output_timeout: 15m
      - run:
          name: Rename the coverage files
          command: |
            mv coverage.xml ocr_coverage.xml
            mv .coverage ocr_coverage

      # Store test results
      - store_test_results:
          path: test-results
      - persist_to_workspace:
          root: .
          paths:
            - ocr_coverage.xml
            - ocr_coverage
  search_testing:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1"
      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
          no_output_timeout: 15m
      - run:
          name: Rename the coverage files
          command: |
            mv coverage.xml search_coverage.xml
            mv .coverage search_coverage

      # Store test results
      - store_test_results:
          path: test-results
      - persist_to_workspace:
          root: .
          paths:
            - search_coverage.xml
            - search_coverage
  # Split litellm_mapped_tests into parallel jobs
  litellm_mapped_tests_proxy_part1:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project
    resource_class: large
    steps:
      - setup_litellm_test_deps
      - run:
          name: Run proxy tests part 1 (high-volume directories)
          command: |
            prisma generate
            export PYTHONUNBUFFERED=1
            python -m pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/client tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part1.xml --durations=10 -n 4 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A
          no_output_timeout: 15m
      - store_test_results:
          path: test-results
  litellm_mapped_tests_proxy_part2:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project
    resource_class: large
    steps:
      - setup_litellm_test_deps
      - run:
          name: Run proxy tests part 2 (all other tests)
          command: |
            prisma generate
            export PYTHONUNBUFFERED=1
            python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 4 --maxfail=5 --timeout=120 -vv --log-cli-level=WARNING -r A
          no_output_timeout: 15m
      - store_test_results:
          path: test-results
  litellm_mapped_enterprise_tests:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project
    resource_class: large

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest-mock==3.12.0"
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-cov==5.0.0"
            pip install "pytest-asyncio==0.21.1"
            pip install "respx==0.22.0"
            pip install "hypercorn==0.17.3"
            pip install "pydantic==2.12.5"
            pip install "mcp==1.26.0"
            pip install "requests-mock>=1.12.1"
            pip install "responses==0.25.7"
            pip install "pytest-xdist==3.6.1"
            pip install "semantic_router==0.1.10" --no-deps
            pip install aurelio_sdk
            pip install "fastapi-offline==1.7.3"
      - setup_litellm_enterprise_pip
      - run:
          name: Run enterprise tests
          command: |
            pwd
            ls
            prisma generate
            python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4
          no_output_timeout: 15m
      # Store test results
      - store_test_results:
          path: test-results
  batches_testing:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "respx==0.22.0"
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-asyncio==0.21.1"
            pip install "pytest-cov==5.0.0"
            pip install "google-generativeai==0.3.2"
            pip install "google-cloud-aiplatform==1.133.0"
            pip install "pytest-xdist==3.6.1"
      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
          no_output_timeout: 15m
      - run:
          name: Rename the coverage files
          command: |
            mv coverage.xml batches_coverage.xml
            mv .coverage batches_coverage

      # Store test results
      - store_test_results:
          path: test-results
      - persist_to_workspace:
          root: .
          paths:
            - batches_coverage.xml
            - batches_coverage
  litellm_utils_testing:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install numpydoc
            pip install "respx==0.22.0"
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-asyncio==0.21.1"
            pip install "pytest-cov==5.0.0"
            pip install "google-generativeai==0.3.2"
            pip install "google-cloud-aiplatform==1.133.0"
            pip install pytest-mock
            pip install "pytest-xdist==3.6.1"
      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
          no_output_timeout: 15m
      - run:
          name: Rename the coverage files
          command: |
            mv coverage.xml litellm_utils_coverage.xml
            mv .coverage litellm_utils_coverage

      # Store test results
      - store_test_results:
          path: test-results
      - persist_to_workspace:
          root: .
          paths:
            - litellm_utils_coverage.xml
            - litellm_utils_coverage

  pass_through_unit_testing:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1"
      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
          no_output_timeout: 15m
      - run:
          name: Rename the coverage files
          command: |
            mv coverage.xml pass_through_unit_tests_coverage.xml
            mv .coverage pass_through_unit_tests_coverage

      # Store test results
      - store_test_results:
          path: test-results
      - persist_to_workspace:
          root: .
          paths:
            - pass_through_unit_tests_coverage.xml
            - pass_through_unit_tests_coverage
  image_gen_testing:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project
    resource_class: large

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-cov==5.0.0"
            pip install "pytest-asyncio==0.21.1"
            pip install "respx==0.22.0"
            pip install "pytest-xdist==3.6.1"
      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5
          no_output_timeout: 15m
      # Store test results
      - store_test_results:
          path: test-results
  logging_testing:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-cov==5.0.0"
            pip install "pytest-asyncio==0.21.1"
            pip install pytest-mock
            pip install "respx==0.22.0"
            pip install "google-generativeai==0.3.2"
            pip install "google-cloud-aiplatform==1.133.0"
            pip install "mlflow==2.17.2"
            pip install "anthropic==0.54.0"
            pip install "blockbuster==1.5.24"
            pip install "pytest-xdist==3.6.1"
            pip install "pytest-timeout==2.2.0"
      # Run pytest and generate JUnit XML report
      - setup_litellm_enterprise_pip
      - run:
          name: Run tests
          command: |
            pwd
            ls
            LITELLM_LOG=WARNING python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread
          no_output_timeout: 15m
      - run:
          name: Rename the coverage files
          command: |
            mv coverage.xml logging_coverage.xml || true
            mv .coverage logging_coverage || true

      # Store test results
      - store_test_results:
          path: test-results
      - persist_to_workspace:
          root: .
          paths:
            - logging_coverage.xml
            - logging_coverage
  audio_testing:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-cov==5.0.0"
            pip install "pytest-asyncio==0.21.1"
            pip install "respx==0.22.0"
      # Run pytest and generate JUnit XML report
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
          no_output_timeout: 15m
      - run:
          name: Rename the coverage files
          command: |
            mv coverage.xml audio_coverage.xml
            mv .coverage audio_coverage

      # Store test results
      - store_test_results:
          path: test-results
      - persist_to_workspace:
          root: .
          paths:
            - audio_coverage.xml
            - audio_coverage
  installing_litellm_on_python:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip
            pip install python-dotenv
            pip install pytest
            pip install tiktoken
            pip install aiohttp
            pip install openai
            pip install click
            pip install "boto3==1.42.80"
            pip install jinja2
            pip install "tokenizers==0.22.2"
            pip install "uvloop==0.21.0"
            pip install "fastuuid==0.14.0"
            pip install jsonschema
      - setup_litellm_enterprise_pip
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -vv tests/local_testing/test_basic_python_version.py

  installing_litellm_on_python_3_13:
    docker:
      - image: cimg/python:3.13.1
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project
    resource_class: medium

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip uv
            pip install wheel setuptools
            uv pip install --system -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-asyncio==0.21.1"
            pip install "pytest-cov==5.0.0"
            pip install "tomli==2.2.1"
            pip install "mcp==1.26.0"
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -v tests/local_testing/test_basic_python_version.py
  helm_chart_testing:
    machine:
      image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker
    resource_class: medium
    working_directory: ~/project

    steps:
      - checkout
      - attach_workspace:
          at: ~/project
      - setup_google_dns
      # Install Helm
      - run:
          name: Install Helm
          command: |
            curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

      # Install kind
      - run:
          name: Install Kind
          command: |
            curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
            chmod +x ./kind
            sudo mv ./kind /usr/local/bin/kind

      # Install kubectl
      - run:
          name: Install kubectl
          command: |
            curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
            chmod +x kubectl
            sudo mv kubectl /usr/local/bin/

      # Create kind cluster
      - run:
          name: Create Kind Cluster
          command: |
            kind create cluster --name litellm-test

      - run:
          name: Load Docker Database Image for helm tests
          command: |
            zstd -d litellm-docker-database.tar.zst --stdout | docker load
            IMAGE_TAG=${CIRCLE_SHA1:-ci}
            docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG}

      - run:
          name: Load Docker image into Kind
          command: |
            IMAGE_TAG=${CIRCLE_SHA1:-ci}
            kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test

      # Run helm lint
      - run:
          name: Run helm lint
          command: |
            helm lint ./deploy/charts/litellm-helm

      # Run helm tests
      - run:
          name: Run helm tests
          command: |
            IMAGE_TAG=${CIRCLE_SHA1:-ci}
            helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \
              --set image.repository=litellm-ci \
              --set image.tag=${IMAGE_TAG} \
              --set image.pullPolicy=Never
            # Wait for pod to be ready
            echo "Waiting 30 seconds for pod to be ready..."
            sleep 30

            # Print pod logs before running tests
            echo "Printing pod logs..."
            kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}")

            # Run the helm tests
            helm test litellm --logs
            helm test litellm --logs

      # Cleanup
      - run:
          name: Cleanup
          command: |
            kind delete cluster --name litellm-test
          when: always # This ensures cleanup runs even if previous steps fail

  check_code_and_doc_quality:
    docker:
      - image: cimg/python:3.11
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    working_directory: ~/project/litellm

    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip
            pip install ruff
            pip install pylint
            pip install pyright
            pip install beautifulsoup4
            pip install .
            curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
      - run: python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
      - run: ruff check ./litellm
      # - run: python ./tests/documentation_tests/test_general_setting_keys.py
      - run: python ./tests/code_coverage_tests/check_licenses.py
      - run: python ./tests/code_coverage_tests/check_provider_folders_documented.py
      - run: python ./tests/code_coverage_tests/router_code_coverage.py
      - run: python ./tests/code_coverage_tests/test_chat_completion_imports.py
      - run: python ./tests/code_coverage_tests/info_log_check.py
      - run: python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py
      - run: python ./tests/code_coverage_tests/test_ban_set_verbose.py
      - run: python ./tests/code_coverage_tests/code_qa_check_tests.py
      - run: python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
      - run: python ./tests/code_coverage_tests/test_proxy_types_import.py
      - run: python ./tests/code_coverage_tests/callback_manager_test.py
      - run: python ./tests/code_coverage_tests/recursive_detector.py
      - run: python ./tests/code_coverage_tests/test_router_strategy_async.py
      - run: python ./tests/code_coverage_tests/litellm_logging_code_coverage.py
      - run: python ./tests/documentation_tests/test_env_keys.py
      - run: python ./tests/documentation_tests/test_router_settings.py
      - run: python ./tests/documentation_tests/test_api_docs.py
      - run: python ./tests/code_coverage_tests/ensure_async_clients_test.py
      - run: python ./tests/code_coverage_tests/enforce_llms_folder_style.py
      - run: python ./tests/documentation_tests/test_circular_imports.py
      - run: python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
      - run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
      - run: python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
      - run: python ./tests/code_coverage_tests/check_fastuuid_usage.py
      - run: python ./tests/code_coverage_tests/memory_test.py
      - run: helm lint ./deploy/charts/litellm-helm

  db_migration_disable_update_check:
    machine:
      image: ubuntu-2204:2023.10.1
    resource_class: medium
    working_directory: ~/project
    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Python 3.9
          command: |
            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
            bash miniconda.sh -b -p $HOME/miniconda
            export PATH="$HOME/miniconda/bin:$PATH"
            conda init bash
            source ~/.bashrc
            conda create -n myenv python=3.9 -y
            conda activate myenv
            python --version
      - run:
          name: Install Dependencies
          command: |
            pip install "pytest==7.3.1"
            pip install "pytest-asyncio==0.21.1"
            pip install aiohttp
            pip install apscheduler
      - run:
          name: Install dockerize
          command: |
            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start PostgreSQL Database
          command: |
            docker run -d \
              --name postgres-db \
              -e POSTGRES_USER=postgres \
              -e POSTGRES_PASSWORD=postgres \
              -e POSTGRES_DB=litellm_test \
              -p 5432:5432 \
              postgres:14
      - run:
          name: Wait for PostgreSQL to be ready
          command: dockerize -wait tcp://localhost:5432 -timeout 1m
      - attach_workspace:
          at: ~/project
      - run:
          name: Load Docker Database Image
          command: |
            zstd -d litellm-docker-database.tar.zst --stdout | docker load
            docker images | grep litellm-docker-database
      - run:
          name: Seed database with real schema
          command: |
            docker run -d \
              -p 4001:4000 \
              -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
              -e LITELLM_MASTER_KEY="sk-1234" \
              --name schema-seed \
              --add-host=host.docker.internal:host-gateway \
              -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
              litellm-docker-database:ci \
              --config /app/config.yaml \
              --port 4000 \
              --use_prisma_db_push
      - run:
          name: Wait for schema seed to complete
          command: dockerize -wait http://localhost:4001 -timeout 5m
      - run:
          name: Stop schema seed container
          command: docker stop schema-seed && docker rm schema-seed
      - run:
          name: Run Docker container with bad schema and disabled updates
          command: |
            docker run -d \
              -p 4000:4000 \
              -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
              -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
              -e DISABLE_SCHEMA_UPDATE="True" \
              --name my-app \
              --add-host=host.docker.internal:host-gateway \
              -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
              -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
              -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \
              litellm-docker-database:ci \
              --config /app/config.yaml \
              --port 4000
      - run:
          name: Wait for container to be ready
          command: dockerize -wait http://localhost:4000 -timeout 1m
      - run:
          name: Check container logs for expected message
          command: |
            echo "=== Printing Full Container Startup Logs ==="
            LOG_OUTPUT="$(docker logs my-app 2>&1)"
            printf '%s\n' "$LOG_OUTPUT"
            echo "=== End of Full Container Startup Logs ==="

            if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then
              echo "Expected message found in logs. Test passed."
            else
              echo "Expected message not found in logs. Test failed."
              exit 1
            fi
      - run:
          name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion)
          command: |
            python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
          no_output_timeout: 15m

  build_and_test:
    machine:
      image: ubuntu-2204:2023.10.1
    resource_class: large
    working_directory: ~/project
    steps:
      - checkout
      - attach_workspace:
          at: ~/project
      - setup_google_dns
      - run:
          name: Install Python 3.9
          command: |
            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
            bash miniconda.sh -b -p $HOME/miniconda
            export PATH="$HOME/miniconda/bin:$PATH"
            conda init bash
            source ~/.bashrc
            conda create -n myenv python=3.9 -y
            conda activate myenv
            python --version
      - run:
          name: Install Dependencies
          command: |
            pip install "pytest==7.3.1"
            pip install "pytest-asyncio==0.21.1"
            pip install aiohttp
            python -m pip install --upgrade pip
            python -m pip install -r .circleci/requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-mock==3.12.0"
            pip install "pytest-asyncio==0.21.1"
            pip install "mypy==1.18.2"
            pip install "google-generativeai==0.3.2"
            pip install "google-cloud-aiplatform==1.133.0"
            pip install pyarrow
            pip install "boto3==1.42.80"
            pip install langchain
            pip install "langfuse>=2.0.0"
            pip install "logfire==0.29.0"
            pip install numpydoc
            pip install prisma
            pip install fastapi
            pip install jsonschema
            pip install "httpx==0.28.1"
            pip install "gunicorn==23.0.0"
            pip install "anyio==4.8.0"
            pip install "aiodynamo==23.10.1"
            pip install "asyncio==3.4.3"
            pip install "PyGithub==1.59.1"
            pip install "openai==1.100.1"
            pip install "litellm[proxy]"
            pip install "pytest-xdist==3.6.1"
      - run:
          name: Install dockerize
          command: |
            wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start PostgreSQL Database
          command: |
            docker run -d \
              --name postgres-db \
              -e POSTGRES_USER=postgres \
              -e POSTGRES_PASSWORD=postgres \
              -e POSTGRES_DB=circle_test \
              -p 5432:5432 \
              postgres:14
      - run:
          name: Wait for PostgreSQL to be ready
          command: dockerize -wait tcp://localhost:5432 -timeout 1m
      - run:
          name: Load Docker Database Image
          command: |
            zstd -d litellm-docker-database.tar.zst --stdout | docker load
            docker tag litellm-docker-database:ci my-app:latest
      - run:
          name: Run Docker container
          command: |
            docker run -d \
              -p 4000:4000 \
              -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
              -e USE_PRISMA_MIGRATE=True \
              -e AZURE_API_KEY=$AZURE_API_KEY \
              -e REDIS_HOST=$REDIS_HOST \
              -e REDIS_PASSWORD=$REDIS_PASSWORD \
              -e REDIS_PORT=$REDIS_PORT \
              -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \
              -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
              -e MISTRAL_API_KEY=$MISTRAL_API_KEY \
              -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
              -e GROQ_API_KEY=$GROQ_API_KEY \
              -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
              -e COHERE_API_KEY=$COHERE_API_KEY \
              -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
              -e AWS_REGION_NAME=$AWS_REGION_NAME \
              -e AUTO_INFER_REGION=True \
              -e OPENAI_API_KEY=$OPENAI_API_KEY \
              -e USE_DDTRACE=True \
              -e DD_API_KEY=$DD_API_KEY \
              -e DD_SITE=$DD_SITE \
              -e LITELLM_LICENSE=$LITELLM_LICENSE \
              -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
              -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
              -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
              -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
              --add-host host.docker.internal:host-gateway \
              --name my-app \
              -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \
              my-app:latest \
              --config /app/config.yaml \
              --port 4000 \
              --detailed_debug \
      - run:
          name: Install curl
          command: |
            sudo apt-get update
            sudo apt-get install -y curl
      - run:
          name: Start outputting logs
          command: docker logs -f my-app
          background: true
      - run:
          name: Wait for app to be ready
          command: dockerize -wait http://localhost:4000 -timeout 5m
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
          no_output_timeout: 15m

      # Store test results
      - store_test_results:
          path: test-results
  e2e_openai_endpoints:
    machine:
      image: ubuntu-2204:2023.10.1
    resource_class: large
    working_directory: ~/project
    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Docker CLI (In case it's not already installed)
          command: |
            curl -fsSL https://get.docker.com | sh
            sudo usermod -aG docker $USER
            docker version
      - run:
          name: Install Python 3.10
          command: |
            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
            bash miniconda.sh -b -p $HOME/miniconda
            export PATH="$HOME/miniconda/bin:$PATH"
            conda init bash
            source ~/.bashrc
            conda create -n myenv python=3.10 -y
            conda activate myenv
            python --version
      - run:
          name: Install Dependencies
          command: |
            pip install "pytest==7.3.1"
            pip install "pytest-asyncio==0.21.1"
            pip install aiohttp
            python -m pip install --upgrade pip
            python -m pip install -r .circleci/requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-mock==3.12.0"
            pip install "pytest-asyncio==0.21.1"
            pip install "mypy==1.18.2"
            pip install "jsonlines==4.0.0"
            pip install "google-generativeai==0.3.2"
            pip install "google-cloud-aiplatform==1.133.0"
            pip install pyarrow
            pip install "boto3==1.42.80"
            pip install langchain
            pip install "langchain_mcp_adapters==0.0.5"
            pip install "langfuse>=2.0.0"
            pip install "logfire==0.29.0"
            pip install numpydoc
            pip install prisma
            pip install fastapi
            pip install jsonschema
            pip install "httpx==0.28.1"
            pip install "gunicorn==23.0.0"
            pip install "anyio==4.8.0"
            pip install "aiodynamo==23.10.1"
            pip install "asyncio==3.4.3"
            pip install "PyGithub==1.59.1"
            pip install "openai==1.100.1"
            # Run pytest and generate JUnit XML report
      - run:
          name: Install dockerize
          command: |
            wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start PostgreSQL Database
          command: |
            docker run -d \
              --name postgres-db \
              -e POSTGRES_USER=postgres \
              -e POSTGRES_PASSWORD=postgres \
              -e POSTGRES_DB=circle_test \
              -p 5432:5432 \
              postgres:14
      - run:
          name: Wait for PostgreSQL to be ready
          command: dockerize -wait tcp://localhost:5432 -timeout 1m
      - attach_workspace:
          at: ~/project
      - run:
          name: Load Docker Database Image
          command: |
            zstd -d litellm-docker-database.tar.zst --stdout | docker load
            docker images | grep litellm-docker-database
      - run:
          name: Run Docker container
          command: |
            docker run -d \
              -p 4000:4000 \
              -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
              -e AZURE_API_KEY=$AZURE_API_KEY \
              -e AZURE_API_BASE=$AZURE_API_BASE \
              -e AZURE_API_VERSION="2024-05-01-preview" \
              -e REDIS_HOST=$REDIS_HOST \
              -e REDIS_PASSWORD=$REDIS_PASSWORD \
              -e REDIS_PORT=$REDIS_PORT \
              -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \
              -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
              -e MISTRAL_API_KEY=$MISTRAL_API_KEY \
              -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
              -e GROQ_API_KEY=$GROQ_API_KEY \
              -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
              -e COHERE_API_KEY=$COHERE_API_KEY \
              -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
              -e AWS_REGION_NAME=$AWS_REGION_NAME \
              -e AUTO_INFER_REGION=True \
              -e USE_DDTRACE=True \
              -e DD_API_KEY=$DD_API_KEY \
              -e DD_SITE=$DD_SITE \
              -e OPENAI_API_KEY=$OPENAI_API_KEY \
              -e LITELLM_LICENSE=$LITELLM_LICENSE \
              -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
              -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
              -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
              -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
              --add-host host.docker.internal:host-gateway \
              --name my-app \
              -v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \
              litellm-docker-database:ci \
              --config /app/config.yaml \
              --port 4000 \
              --detailed_debug \
      - run:
          name: Install curl
          command: |
            sudo apt-get update
            sudo apt-get install -y curl
      - run:
          name: Start outputting logs
          command: docker logs -f my-app
          background: true
      - run:
          name: Wait for app to be ready
          command: dockerize -wait http://localhost:4000 -timeout 5m
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5
          no_output_timeout: 15m

      # Store test results
      - store_test_results:
          path: test-results
  proxy_logging_guardrails_model_info_tests:
    machine:
      image: ubuntu-2204:2023.10.1
    resource_class: large
    working_directory: ~/project
    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Docker CLI (In case it's not already installed)
          command: |
            curl -fsSL https://get.docker.com | sh
            sudo usermod -aG docker $USER
            docker version
      - run:
          name: Install Python 3.9
          command: |
            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
            bash miniconda.sh -b -p $HOME/miniconda
            export PATH="$HOME/miniconda/bin:$PATH"
            conda init bash
            source ~/.bashrc
            conda create -n myenv python=3.9 -y
            conda activate myenv
            python --version
      - run:
          name: Install Dependencies
          command: |
            pip install "pytest==7.3.1"
            pip install "pytest-asyncio==0.21.1"
            pip install aiohttp
            python -m pip install --upgrade pip
            python -m pip install -r .circleci/requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-mock==3.12.0"
            pip install "pytest-asyncio==0.21.1"
            pip install "mypy==1.18.2"
            pip install "google-generativeai==0.3.2"
            pip install "google-cloud-aiplatform==1.133.0"
            pip install pyarrow
            pip install "boto3==1.42.80"
            pip install langchain
            pip install "langfuse>=2.0.0"
            pip install "logfire==0.29.0"
            pip install numpydoc
            pip install prisma
            pip install fastapi
            pip install jsonschema
            pip install "httpx==0.28.1"
            pip install "gunicorn==23.0.0"
            pip install "anyio==4.8.0"
            pip install "aiodynamo==23.10.1"
            pip install "asyncio==3.4.3"
            pip install "PyGithub==1.59.1"
            pip install "openai==1.100.1"
      - run:
          name: Install dockerize
          command: |
            wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start PostgreSQL Database
          command: |
            docker run -d \
              --name postgres-db \
              -e POSTGRES_USER=postgres \
              -e POSTGRES_PASSWORD=postgres \
              -e POSTGRES_DB=circle_test \
              -p 5432:5432 \
              postgres:14
      - run:
          name: Wait for PostgreSQL to be ready
          command: dockerize -wait tcp://localhost:5432 -timeout 1m
      - attach_workspace:
          at: ~/project
      - run:
          name: Load Docker Database Image
          command: |
            zstd -d litellm-docker-database.tar.zst --stdout | docker load
            docker images | grep litellm-docker-database
      - run:
          name: Run Docker container
          # intentionally give bad redis credentials here
          # the OTEL test - should get this as a trace
          command: |
            docker run -d \
              -p 4000:4000 \
              -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
              -e REDIS_HOST=$REDIS_HOST \
              -e REDIS_PASSWORD=$REDIS_PASSWORD \
              -e REDIS_PORT=$REDIS_PORT \
              -e LITELLM_MASTER_KEY="sk-1234" \
              -e OPENAI_API_KEY=$OPENAI_API_KEY \
              -e LITELLM_LICENSE=$LITELLM_LICENSE \
              -e OTEL_EXPORTER="in_memory" \
              -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
              -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
              -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
              -e USE_DDTRACE=True \
              -e DD_API_KEY=$DD_API_KEY \
              -e DD_SITE=$DD_SITE \
              -e AWS_REGION_NAME=$AWS_REGION_NAME \
              -e COHERE_API_KEY=$COHERE_API_KEY \
              -e GCS_FLUSH_INTERVAL="1" \
              --add-host host.docker.internal:host-gateway \
              --name my-app \
              -v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \
              -v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \
              litellm-docker-database:ci \
              --config /app/config.yaml \
              --port 4000 \
              --detailed_debug \
      - run:
          name: Install curl and dockerize
          command: |
            sudo apt-get update
            sudo apt-get install -y curl
            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start outputting logs
          command: docker logs -f my-app
          background: true
      - run:
          name: Wait for app to be ready
          command: dockerize -wait http://localhost:4000 -timeout 5m
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -v tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5
          no_output_timeout: 15m
            # Clean up first container
      - run:
          name: Stop and remove first container
          command: |
            docker stop my-app
            docker rm my-app

      # Second Docker Container Run with Different Config
      # NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license
      - run:
          name: Run Second Docker container
          command: |
            docker run -d \
              -p 4000:4000 \
              -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
              -e REDIS_HOST=$REDIS_HOST \
              -e REDIS_PASSWORD=$REDIS_PASSWORD \
              -e REDIS_PORT=$REDIS_PORT \
              -e LITELLM_MASTER_KEY="sk-1234" \
              -e OPENAI_API_KEY=$OPENAI_API_KEY \
              -e LITELLM_LICENSE="bad-license" \
              --add-host host.docker.internal:host-gateway \
              --name my-app-3 \
              -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \
              litellm-docker-database:ci \
              --config /app/config.yaml \
              --port 4000 \
              --detailed_debug

      - run:
          name: Start outputting logs for second container
          command: docker logs -f my-app-2
          background: true

      - run:
          name: Wait for second app to be ready
          command: dockerize -wait http://localhost:4000 -timeout 5m

      - run:
          name: Run second round of tests
          command: |
            python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
          no_output_timeout: 15m

      # Store test results
      - store_test_results:
          path: test-results
  proxy_spend_accuracy_tests:
    machine:
      image: ubuntu-2204:2023.10.1
    resource_class: large
    working_directory: ~/project
    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Docker CLI (In case it's not already installed)
          command: |
            curl -fsSL https://get.docker.com | sh
            sudo usermod -aG docker $USER
            docker version
      - run:
          name: Install Python 3.9
          command: |
            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
            bash miniconda.sh -b -p $HOME/miniconda
            export PATH="$HOME/miniconda/bin:$PATH"
            conda init bash
            source ~/.bashrc
            conda create -n myenv python=3.9 -y
            conda activate myenv
            python --version
      - run:
          name: Install Dependencies
          command: |
            pip install "pytest==7.3.1"
            pip install "pytest-asyncio==0.21.1"
            pip install aiohttp
            python -m pip install --upgrade pip
            python -m pip install -r requirements.txt
      - run:
          name: Install dockerize
          command: |
            wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start PostgreSQL Database
          command: |
            docker run -d \
              --name postgres-db \
              -e POSTGRES_USER=postgres \
              -e POSTGRES_PASSWORD=postgres \
              -e POSTGRES_DB=circle_test \
              -p 5432:5432 \
              postgres:14
      - run:
          name: Wait for PostgreSQL to be ready
          command: dockerize -wait tcp://localhost:5432 -timeout 1m
      - attach_workspace:
          at: ~/project
      - run:
          name: Load Docker Database Image
          command: |
            zstd -d litellm-docker-database.tar.zst --stdout | docker load
            docker images | grep litellm-docker-database
      - run:
          name: Run Docker container
          # intentionally give bad redis credentials here
          # the OTEL test - should get this as a trace
          command: |
            docker run -d \
              -p 4000:4000 \
              -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
              -e REDIS_HOST=$REDIS_HOST \
              -e REDIS_PASSWORD=$REDIS_PASSWORD \
              -e REDIS_PORT=$REDIS_PORT \
              -e LITELLM_MASTER_KEY="sk-1234" \
              -e OPENAI_API_KEY=$OPENAI_API_KEY \
              -e LITELLM_LICENSE=$LITELLM_LICENSE \
              -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
              -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
              -e USE_DDTRACE=True \
              -e DD_API_KEY=$DD_API_KEY \
              -e DD_SITE=$DD_SITE \
              -e AWS_REGION_NAME=$AWS_REGION_NAME \
              --add-host host.docker.internal:host-gateway \
              --name my-app \
              -v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \
              litellm-docker-database:ci \
              --config /app/config.yaml \
              --port 4000 \
              --detailed_debug \
      - run:
          name: Install curl
          command: |
            sudo apt-get update
            sudo apt-get install -y curl
      - run:
          name: Start outputting logs
          command: docker logs -f my-app
          background: true
      - run:
          name: Wait for app to be ready
          command: dockerize -wait http://localhost:4000 -timeout 5m
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5
          no_output_timeout: 15m
            # Clean up first container
      - run:
          name: Stop and remove first container
          command: |
            docker stop my-app
            docker rm my-app

  proxy_multi_instance_tests:
    machine:
      image: ubuntu-2204:2023.10.1
    resource_class: large
    working_directory: ~/project
    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Docker CLI (In case it's not already installed)
          command: |
            curl -fsSL https://get.docker.com | sh
            sudo usermod -aG docker $USER
            docker version
      - run:
          name: Install Python 3.9
          command: |
            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
            bash miniconda.sh -b -p $HOME/miniconda
            export PATH="$HOME/miniconda/bin:$PATH"
            conda init bash
            source ~/.bashrc
            conda create -n myenv python=3.9 -y
            conda activate myenv
            python --version
      - run:
          name: Install Dependencies
          command: |
            pip install "pytest==7.3.1"
            pip install "pytest-asyncio==0.21.1"
            pip install aiohttp
            python -m pip install --upgrade pip
            python -m pip install -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-mock==3.12.0"
            pip install "pytest-asyncio==0.21.1"
      - run:
          name: Install dockerize
          command: |
            wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start PostgreSQL Database
          command: |
            docker run -d \
              --name postgres-db \
              -e POSTGRES_USER=postgres \
              -e POSTGRES_PASSWORD=postgres \
              -e POSTGRES_DB=circle_test \
              -p 5432:5432 \
              postgres:14
      - run:
          name: Wait for PostgreSQL to be ready
          command: dockerize -wait tcp://localhost:5432 -timeout 1m
      - attach_workspace:
          at: ~/project
      - run:
          name: Load Docker Database Image
          command: |
            zstd -d litellm-docker-database.tar.zst --stdout | docker load
            docker images | grep litellm-docker-database
      - run:
          name: Run Docker container 1
          # intentionally give bad redis credentials here
          # the OTEL test - should get this as a trace
          command: |
            docker run -d \
              -p 4000:4000 \
              -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
              -e REDIS_HOST=$REDIS_HOST \
              -e REDIS_PASSWORD=$REDIS_PASSWORD \
              -e REDIS_PORT=$REDIS_PORT \
              -e LITELLM_MASTER_KEY="sk-1234" \
              -e LITELLM_LICENSE=$LITELLM_LICENSE \
              -e USE_DDTRACE=True \
              -e DD_API_KEY=$DD_API_KEY \
              -e DD_SITE=$DD_SITE \
              --add-host host.docker.internal:host-gateway \
              --name my-app \
              -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
              litellm-docker-database:ci \
              --config /app/config.yaml \
              --port 4000 \
              --detailed_debug \
      - run:
          name: Run Docker container 2
          command: |
            docker run -d \
              -p 4001:4001 \
              -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
              -e REDIS_HOST=$REDIS_HOST \
              -e REDIS_PASSWORD=$REDIS_PASSWORD \
              -e REDIS_PORT=$REDIS_PORT \
              -e LITELLM_MASTER_KEY="sk-1234" \
              -e LITELLM_LICENSE=$LITELLM_LICENSE \
              -e USE_DDTRACE=True \
              -e DD_API_KEY=$DD_API_KEY \
              -e DD_SITE=$DD_SITE \
              --add-host host.docker.internal:host-gateway \
              --name my-app-2 \
              -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
              litellm-docker-database:ci \
              --config /app/config.yaml \
              --port 4001 \
              --detailed_debug
      - run:
          name: Install curl and dockerize
          command: |
            sudo apt-get update
            sudo apt-get install -y curl
            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start outputting logs
          command: docker logs -f my-app
          background: true
      - run:
          name: Wait for instance 1 to be ready
          command: dockerize -wait http://localhost:4000 -timeout 5m
      - run:
          name: Wait for instance 2 to be ready
          command: dockerize -wait http://localhost:4001 -timeout 5m
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5
          no_output_timeout: 15m
            # Clean up first container
      # Store test results
      - store_test_results:
          path: test-results

  proxy_store_model_in_db_tests:
    machine:
      image: ubuntu-2204:2023.10.1
    resource_class: large
    working_directory: ~/project
    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Docker CLI (In case it's not already installed)
          command: |
            curl -fsSL https://get.docker.com | sh
            sudo usermod -aG docker $USER
            docker version
            sudo systemctl restart docker
      - run:
          name: Install Python 3.9
          command: |
            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
            bash miniconda.sh -b -p $HOME/miniconda
            export PATH="$HOME/miniconda/bin:$PATH"
            conda init bash
            source ~/.bashrc
            conda create -n myenv python=3.9 -y
            conda activate myenv
            python --version
      - run:
          name: Install Dependencies
          command: |
            pip install "pytest==7.3.1"
            pip install "pytest-asyncio==0.21.1"
            pip install aiohttp
            python -m pip install --upgrade pip
            python -m pip install -r requirements.txt
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-mock==3.12.0"
            pip install "pytest-asyncio==0.21.1"
            pip install "assemblyai==0.37.0"
      - run:
          name: Install dockerize
          command: |
            wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start PostgreSQL Database
          command: |
            docker run -d \
              --name postgres-db \
              -e POSTGRES_USER=postgres \
              -e POSTGRES_PASSWORD=postgres \
              -e POSTGRES_DB=circle_test \
              -p 5432:5432 \
              postgres:14
      - run:
          name: Wait for PostgreSQL to be ready
          command: dockerize -wait tcp://localhost:5432 -timeout 1m
      - attach_workspace:
          at: ~/project
      - run:
          name: Load Docker Database Image
          command: |
            zstd -d litellm-docker-database.tar.zst --stdout | docker load
            docker images | grep litellm-docker-database
      - run:
          name: Run Docker container
          # intentionally give bad redis credentials here
          # the OTEL test - should get this as a trace
          command: |
            docker run -d \
              -p 4000:4000 \
              -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
              -e STORE_MODEL_IN_DB="True" \
              -e LITELLM_MASTER_KEY="sk-1234" \
              -e LITELLM_LICENSE=$LITELLM_LICENSE \
              --add-host host.docker.internal:host-gateway \
              --name my-app \
              -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
              litellm-docker-database:ci \
              --config /app/config.yaml \
              --port 4000 \
              --detailed_debug \
      - run:
          name: Install curl and dockerize
          command: |
            sudo apt-get update
            sudo apt-get install -y curl
            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start outputting logs
          command: docker logs -f my-app
          background: true
      - run:
          name: Wait for app to be ready
          command: dockerize -wait http://localhost:4000 -timeout 5m
      - run:
          name: Run tests
          command: |
            pwd
            ls
            python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5
          no_output_timeout: 15m
      - run:
          name: Stop and remove containers
          command: |
            docker stop my-app || true
            docker rm my-app || true
            docker stop postgres-db || true
            docker rm postgres-db || true
          when: always
      - store_test_results:
          path: test-results

  proxy_build_from_pip_tests:
    # Change from docker to machine executor
    machine:
      image: ubuntu-2204:2023.10.1
    resource_class: large
    working_directory: ~/project
    steps:
      - checkout
      - setup_google_dns
      # Remove Docker CLI installation since it's already available in machine executor
      - run:
          name: Install Python 3.13
          command: |
            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
            bash miniconda.sh -b -p $HOME/miniconda
            export PATH="$HOME/miniconda/bin:$PATH"
            conda init bash
            source ~/.bashrc
            conda create -n myenv python=3.13 -y
            conda activate myenv
            python --version
      - run:
          name: Install Dependencies
          command: |
            python -m pip install --upgrade pip
            pip install "pytest==7.3.1" "pytest-asyncio==0.21.1" "pytest-retry==1.6.3" \
              "pytest-mock==3.12.0" "mypy==1.18.2" aiohttp apscheduler
      - run:
          name: Build Docker image
          command: |
            docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip .
      - run:
          name: Start PostgreSQL Database
          command: |
            docker run -d \
              --name postgres-db \
              -e POSTGRES_USER=postgres \
              -e POSTGRES_PASSWORD=postgres \
              -e POSTGRES_DB=circle_test \
              -p 5432:5432 \
              postgres:14
      - run:
          name: Wait for PostgreSQL to be ready
          command: |
            timeout 60s bash -c 'until docker exec postgres-db pg_isready -U postgres -d circle_test; do sleep 2; done'
      - run:
          name: Run Docker container
          # intentionally give bad redis credentials here
          # the OTEL test - should get this as a trace
          command: |
            docker run -d \
              -p 4000:4000 \
              -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
              -e REDIS_HOST=$REDIS_HOST \
              -e REDIS_PASSWORD=$REDIS_PASSWORD \
              -e REDIS_PORT=$REDIS_PORT \
              -e LITELLM_MASTER_KEY="sk-1234" \
              -e OPENAI_API_KEY=$OPENAI_API_KEY \
              -e LITELLM_LICENSE=$LITELLM_LICENSE \
              -e OTEL_EXPORTER="in_memory" \
              -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
              -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
              -e AWS_REGION_NAME=$AWS_REGION_NAME \
              -e COHERE_API_KEY=$COHERE_API_KEY \
              -e USE_DDTRACE=True \
              -e DD_API_KEY=$DD_API_KEY \
              -e DD_SITE=$DD_SITE \
              -e GCS_FLUSH_INTERVAL="1" \
              --add-host host.docker.internal:host-gateway \
              --name my-app \
              -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \
              my-app:latest \
              --config /app/config.yaml \
              --port 4000 \
              --detailed_debug \
      - run:
          name: Install curl and dockerize
          command: |
            sudo apt-get update
            sudo apt-get install -y curl
            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start outputting logs
          command: docker logs -f my-app
          background: true
      - run:
          name: Wait for app to be ready
          command: dockerize -wait http://localhost:4000 -timeout 5m
      - run:
          name: Run tests
          command: |
            python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
          no_output_timeout: 15m
            # Clean up first container
      - run:
          name: Stop and remove first container
          command: |
            docker stop my-app || true
            docker rm my-app || true
            docker stop postgres-db || true
            docker rm postgres-db || true
          when: always
  proxy_pass_through_endpoint_tests:
    machine:
      image: ubuntu-2204:2023.10.1
    resource_class: large
    working_directory: ~/project
    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Python 3.10
          command: |
            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
            bash miniconda.sh -b -p $HOME/miniconda
            export PATH="$HOME/miniconda/bin:$PATH"
            conda init bash
            source ~/.bashrc
            conda create -n myenv python=3.10 -y
            conda activate myenv
            python --version
      - run:
          name: Install Dependencies
          command: |
            export PATH="$HOME/miniconda/bin:$PATH"
            source $HOME/miniconda/etc/profile.d/conda.sh
            conda activate myenv
            pip install "pytest==7.3.1"
            pip install "pytest-retry==1.6.3"
            pip install "pytest-asyncio==0.21.1"
            pip install "google-cloud-aiplatform==1.133.0"
            pip install aiohttp
            pip install "openai==1.100.1"
            pip install "assemblyai==0.37.0"
            python -m pip install --upgrade pip
            pip install "pydantic==2.12.5"
            pip install "pytest==7.3.1"
            pip install "pytest-mock==3.12.0"
            pip install "pytest-asyncio==0.21.1"
            pip install "boto3==1.42.80"
            pip install "mypy==1.18.2"
            pip install pyarrow
            pip install numpydoc
            pip install prisma
            pip install fastapi
            pip install jsonschema
            pip install "httpx==0.27.0"
            pip install "anyio==4.8.0"
            pip install "asyncio==3.4.3"
            pip install "PyGithub==1.59.1"
            pip install "google-cloud-aiplatform==1.59.0"
            pip install "anthropic==0.54.0"
            pip install "langchain_mcp_adapters==0.0.5"
            pip install "langchain_openai==0.2.1"
            pip install "langgraph==0.3.18"
            pip install "fastuuid==0.13.5"
            pip install -r requirements.txt
      - run:
          name: Install dockerize
          command: |
            wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start PostgreSQL Database
          command: |
            docker run -d \
              --name postgres-db \
              -e POSTGRES_USER=postgres \
              -e POSTGRES_PASSWORD=postgres \
              -e POSTGRES_DB=circle_test \
              -p 5432:5432 \
              postgres:14
      - run:
          name: Wait for PostgreSQL to be ready
          command: dockerize -wait tcp://localhost:5432 -timeout 1m
      - attach_workspace:
          at: ~/project
      - run:
          name: Load Docker Database Image
          command: |
            zstd -d litellm-docker-database.tar.zst --stdout | docker load
            docker images | grep litellm-docker-database
      - run:
          name: Run Docker container
          command: |
            docker run -d \
              -p 4000:4000 \
              -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
              -e LITELLM_MASTER_KEY="sk-1234" \
              -e OPENAI_API_KEY=$OPENAI_API_KEY \
              -e GEMINI_API_KEY=$GEMINI_API_KEY \
              -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
              -e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \
              -e AZURE_API_KEY=$AZURE_API_KEY \
              -e AZURE_API_BASE=$AZURE_API_BASE \
              -e USE_DDTRACE=True \
              -e DD_API_KEY=$DD_API_KEY \
              -e DD_SITE=$DD_SITE \
              -e LITELLM_LICENSE=$LITELLM_LICENSE \
              -e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \
              --add-host host.docker.internal:host-gateway \
              --name my-app \
              -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
              -v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \
              litellm-docker-database:ci \
              --config /app/config.yaml \
              --port 4000 \
              --detailed_debug \
      - run:
          name: Start outputting logs
          command: docker logs -f my-app
          background: true
      - run:
          name: Wait for app to be ready
          command: dockerize -wait http://localhost:4000 -timeout 5m
      # Add Ruby installation and testing before the existing Node.js and Python tests
      - run:
          name: Install Ruby and Bundler
          command: |
            # Import GPG keys first
            gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB || {
              curl -sSL https://rvm.io/mpapis.asc | gpg --import -
              curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -
            }

            # Install Ruby version manager (RVM)
            curl -sSL https://get.rvm.io | bash -s stable

            # Source RVM from the correct location
            source $HOME/.rvm/scripts/rvm

            # Install Ruby 3.2.2
            rvm install 3.2.2
            rvm use 3.2.2 --default

            # Install latest Bundler
            gem install bundler

      - run:
          name: Run Ruby tests
          command: |
            source $HOME/.rvm/scripts/rvm
            cd tests/pass_through_tests/ruby_passthrough_tests
            bundle install
            bundle exec rspec
          no_output_timeout: 30m
      # New steps to run Node.js test
      - run:
          name: Install Node.js
          command: |
            export DEBIAN_FRONTEND=noninteractive
            curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash -
            sudo apt-get update
            sudo apt-get install -y nodejs
            node --version
            npm --version

      - run:
          name: Install Node.js dependencies
          command: |
            npm install @google-cloud/vertexai
            npm install @google/generative-ai
            npm install --save-dev jest

      - run:
          name: Run Vertex AI, Google AI Studio Node.js tests
          command: |
            npx jest tests/pass_through_tests --verbose
          no_output_timeout: 30m
      - run:
          name: Run tests
          command: |
            export PATH="$HOME/miniconda/bin:$PATH"
            source $HOME/miniconda/etc/profile.d/conda.sh
            conda activate myenv
            pwd
            ls
            python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5
          no_output_timeout: 15m

      # Store test results
      - store_test_results:
          path: test-results

  proxy_e2e_anthropic_messages_tests:
    machine:
      image: ubuntu-2204:2023.10.1
    resource_class: large
    working_directory: ~/project
    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Docker CLI (In case it's not already installed)
          command: |
            curl -fsSL https://get.docker.com | sh
            sudo usermod -aG docker $USER
            docker version
      - run:
          name: Install Python 3.10
          command: |
            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
            bash miniconda.sh -b -p $HOME/miniconda
            export PATH="$HOME/miniconda/bin:$PATH"
            conda init bash
            source ~/.bashrc
            conda create -n myenv python=3.10 -y
            conda activate myenv
            python --version
      - run:
          name: Install Dependencies
          command: |
            export PATH="$HOME/miniconda/bin:$PATH"
            source $HOME/miniconda/etc/profile.d/conda.sh
            conda activate myenv
            pip install "pytest==7.3.1"
            pip install "pytest-asyncio==0.21.1"
            pip install "boto3==1.42.80"
            pip install "httpx==0.27.0"
            pip install "claude-agent-sdk"
            pip install -r requirements.txt
      - run:
          name: Install dockerize
          command: |
            wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start PostgreSQL Database
          command: |
            docker run -d \
              --name postgres-db \
              -e POSTGRES_USER=postgres \
              -e POSTGRES_PASSWORD=postgres \
              -e POSTGRES_DB=circle_test \
              -p 5432:5432 \
              postgres:14
      - run:
          name: Wait for PostgreSQL to be ready
          command: dockerize -wait tcp://localhost:5432 -timeout 1m
      - attach_workspace:
          at: ~/project
      - run:
          name: Load Docker Database Image
          command: |
            zstd -d litellm-docker-database.tar.zst --stdout | docker load
            docker images | grep litellm-docker-database
      - run:
          name: Run Docker container with test config
          command: |
            docker run -d \
              -p 4000:4000 \
              -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
              -e LITELLM_MASTER_KEY="sk-1234" \
              -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
              -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
              -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
              -e AWS_REGION_NAME="us-east-1" \
              -e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \
              --add-host host.docker.internal:host-gateway \
              --name my-app \
              -v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \
              litellm-docker-database:ci \
              --config /app/config.yaml \
              --port 4000 \
              --detailed_debug
      - run:
          name: Start outputting logs
          command: docker logs -f my-app
          background: true
      - run:
          name: Wait for app to be ready
          command: dockerize -wait http://localhost:4000 -timeout 5m
      - run:
          name: Run Claude Agent SDK E2E Tests
          command: |
            export PATH="$HOME/miniconda/bin:$PATH"
            source $HOME/miniconda/etc/profile.d/conda.sh
            conda activate myenv
            export LITELLM_PROXY_URL="http://localhost:4000"
            export LITELLM_API_KEY="sk-1234"
            pwd
            ls
            python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5
          no_output_timeout: 15m

      # Store test results
      - store_test_results:
          path: test-results

  proxy_e2e_azure_batches_tests:
    machine:
      image: ubuntu-2204:2023.10.1
    resource_class: large
    working_directory: ~/project
    steps:
      - checkout
      - setup_google_dns
      - run:
          name: Install Docker CLI
          command: |
            curl -fsSL https://get.docker.com | sh
            sudo usermod -aG docker $USER
            docker version
      - run:
          name: Install Python 3.12
          command: |
            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
            bash miniconda.sh -b -p $HOME/miniconda
            export PATH="$HOME/miniconda/bin:$PATH"
            conda init bash
            source ~/.bashrc
            conda create -n myenv python=3.12 -y
            conda activate myenv
            python --version
      - run:
          name: Install Poetry
          command: |
            export PATH="$HOME/miniconda/bin:$PATH"
            source $HOME/miniconda/etc/profile.d/conda.sh
            conda activate myenv
            pip install poetry
      - run:
          name: Install dockerize
          command: |
            wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start PostgreSQL Database
          command: |
            docker run -d \
              --name postgres-db \
              -e POSTGRES_USER=llmproxy \
              -e POSTGRES_PASSWORD=dbpassword9090 \
              -e POSTGRES_DB=litellm \
              -p 5432:5432 \
              postgres:15
      - run:
          name: Wait for PostgreSQL to be ready
          command: dockerize -wait tcp://localhost:5432 -timeout 1m
      - run:
          name: Install system dependencies
          command: |
            sudo apt-get update -y
            sudo apt-get install -y libpq-dev
      - run:
          name: Install Dependencies
          command: |
            export PATH="$HOME/miniconda/bin:$PATH"
            source $HOME/miniconda/etc/profile.d/conda.sh
            conda activate myenv
            poetry config virtualenvs.in-project true
            poetry install --with dev,proxy-dev --extras "proxy"
            poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity
      - run:
          name: Setup litellm-enterprise
          command: |
            export PATH="$HOME/miniconda/bin:$PATH"
            source $HOME/miniconda/etc/profile.d/conda.sh
            conda activate myenv
            poetry run pip install --force-reinstall --no-deps -e enterprise/
      - run:
          name: Generate Prisma client
          command: |
            export PATH="$HOME/miniconda/bin:$PATH"
            source $HOME/miniconda/etc/profile.d/conda.sh
            conda activate myenv
            poetry run prisma generate --schema litellm/proxy/schema.prisma
      - run:
          name: Run Prisma migrations
          command: |
            export PATH="$HOME/miniconda/bin:$PATH"
            source $HOME/miniconda/etc/profile.d/conda.sh
            conda activate myenv
            export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
            cd litellm/proxy
            poetry run prisma migrate deploy --schema schema.prisma
            cd ../..
      - run:
          name: Run Azure Batch E2E Tests
          command: |
            export PATH="$HOME/miniconda/bin:$PATH"
            source $HOME/miniconda/etc/profile.d/conda.sh
            conda activate myenv
            export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
            export USE_LOCAL_LITELLM=true
            export USE_MOCK_MODELS=true
            export USE_STATE_TRACKER=true
            export LITELLM_LOG=DEBUG
            poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
              -vv -s -k "test_e2e_managed_batch" \
              --tb=short \
              --maxfail=3 \
              --durations=10 \
              --junitxml=test-results/junit.xml
          no_output_timeout: 15m

  upload-coverage:
    docker:
      - image: cimg/python:3.9
    steps:
      - checkout
      - attach_workspace:
          at: .
      # Check file locations
      - run:
          name: Check coverage file location
          command: |
            echo "Current directory:"
            ls -la
            echo "\nContents of tests/llm_translation:"
            ls -la tests/llm_translation
      - run:
          name: Combine Coverage
          command: |
            python -m venv venv
            . venv/bin/activate
            pip install coverage
            coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage
            coverage xml
      - codecov/upload:
          file: ./coverage.xml

  publish_proxy_extras:
    docker:
      - image: cimg/python:3.8
    working_directory: ~/project/litellm-proxy-extras
    environment:
      TWINE_USERNAME: __token__

    steps:
      - checkout:
          path: ~/project

      - run:
          name: Check if litellm-proxy-extras dir or pyproject.toml was modified
          command: |
            echo "Install TOML package."
            python -m pip install toml
            # Get current version from pyproject.toml
            CURRENT_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")

            # Get last published version from PyPI
            LAST_VERSION=$(curl -s https://pypi.org/pypi/litellm-proxy-extras/json | python -c "import json, sys; print(json.load(sys.stdin)['info']['version'])")

            echo "Current version: $CURRENT_VERSION"
            echo "Last published version: $LAST_VERSION"

            # Compare versions using Python's packaging.version
            VERSION_COMPARE=$(python -c "from packaging import version; print(1 if version.parse('$CURRENT_VERSION') < version.parse('$LAST_VERSION') else 0)")

            echo "Version compare: $VERSION_COMPARE"
            if [ "$VERSION_COMPARE" = "1" ]; then
              echo "Error: Current version ($CURRENT_VERSION) is less than last published version ($LAST_VERSION)"
              exit 1
            fi

            # If versions are equal or current is greater, check contents
            pip download --no-deps litellm-proxy-extras==$LAST_VERSION -d /tmp

            echo "Contents of /tmp directory:"
            ls -la /tmp

            # Find the downloaded file (could be .whl or .tar.gz)
            DOWNLOADED_FILE=$(ls /tmp/litellm_proxy_extras-*)
            echo "Downloaded file: $DOWNLOADED_FILE"

            # Extract based on file extension
            if [[ "$DOWNLOADED_FILE" == *.whl ]]; then
                echo "Extracting wheel file..."
                unzip -q "$DOWNLOADED_FILE" -d /tmp/extracted
                EXTRACTED_DIR="/tmp/extracted"
            else
                echo "Extracting tar.gz file..."
                tar -xzf "$DOWNLOADED_FILE" -C /tmp
                EXTRACTED_DIR="/tmp/litellm_proxy_extras-$LAST_VERSION"
            fi

            echo "Contents of extracted package:"
            ls -R "$EXTRACTED_DIR"

            # Compare contents
            if ! diff -r "$EXTRACTED_DIR/litellm_proxy_extras" ./litellm_proxy_extras; then
              if [ "$CURRENT_VERSION" = "$LAST_VERSION" ]; then
                echo "Error: Changes detected in litellm-proxy-extras but version was not bumped"
                echo "Current version: $CURRENT_VERSION"
                echo "Last published version: $LAST_VERSION"
                echo "Changes:"
                diff -r "$EXTRACTED_DIR/litellm_proxy_extras" ./litellm_proxy_extras
                exit 1
              fi
            else
              echo "No changes detected in litellm-proxy-extras. Skipping PyPI publish."
              circleci step halt
            fi

      - run:
          name: Get new version
          command: |
            NEW_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
            echo "export NEW_VERSION=$NEW_VERSION" >> $BASH_ENV

      - run:
          name: Check if versions match
          command: |
            cd ~/project
            # Check pyproject.toml
            CURRENT_VERSION=$(python -c "import toml; dep = toml.load('pyproject.toml')['tool']['poetry']['dependencies']['litellm-proxy-extras']; print(dep['version'] if isinstance(dep, dict) else dep)")
            if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then
              echo "Error: Version in pyproject.toml ($CURRENT_VERSION) doesn't match new version ($NEW_VERSION)"
              exit 1
            fi

            # Check requirements.txt
            REQ_VERSION=$(grep -oP 'litellm-proxy-extras==\K[0-9.]+' requirements.txt)
            if [ "$REQ_VERSION" != "$NEW_VERSION" ]; then
              echo "Error: Version in requirements.txt ($REQ_VERSION) doesn't match new version ($NEW_VERSION)"
              exit 1
            fi

      - run:
          name: Publish to PyPI
          command: |
            echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc
            python -m pip install --upgrade pip build twine setuptools wheel
            rm -rf build dist
            python -m build
            twine upload --verbose dist/*

  ui_build:
    docker:
      - image: cimg/node:20.19
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    resource_class: medium+
    working_directory: ~/project
    steps:
      - checkout
      - setup_google_dns
      - restore_cache:
          keys:
            - ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
            - ui-build-deps-v1-
      - restore_cache:
          keys:
            - ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
            - ui-nextjs-cache-v1-
      - run:
          name: Install dependencies
          command: |
            cd ui/litellm-dashboard
            npm ci
      - save_cache:
          key: ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
          paths:
            - ui/litellm-dashboard/node_modules
      - run:
          name: Build UI
          command: |
            cd ui/litellm-dashboard
            source ./build_ui.sh
      - save_cache:
          key: ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
          paths:
            - ui/litellm-dashboard/.next/cache
      - persist_to_workspace:
          root: .
          paths:
            - litellm/proxy/_experimental/out

  ui_unit_tests:
    docker:
      - image: cimg/node:20.19
        auth:
          username: ${DOCKERHUB_USERNAME}
          password: ${DOCKERHUB_PASSWORD}
    resource_class: xlarge
    working_directory: ~/project
    steps:
      - checkout
      - setup_google_dns
      - restore_cache:
          keys:
            - ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
            - ui-unit-deps-v1-
      - run:
          name: Install dependencies
          command: |
            cd ui/litellm-dashboard
            npm ci
      - save_cache:
          key: ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
          paths:
            - ui/litellm-dashboard/node_modules
      - run:
          name: Run UI unit tests (Vitest)
          command: |
            cd ui/litellm-dashboard

            CI=true npm run test -- --run \
              --pool forks --poolOptions.forks.maxForks=8

  build_docker_database_image:
    machine:
      image: ubuntu-2204:2024.04.1
    resource_class: large
    working_directory: ~/project
    steps:
      - checkout

      - run:
          name: Build Docker image
          command: |
            docker build \
              -t litellm-docker-database:ci \
              -f docker/Dockerfile.database .

      - run:
          name: Save Docker image to workspace root
          command: |
            docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst

      - persist_to_workspace:
          root: .
          paths:
            - litellm-docker-database.tar.zst

  e2e_ui_testing:
    machine:
      image: ubuntu-2204:2023.10.1
    resource_class: large
    working_directory: ~/project
    parameters:
      browser:
        type: string
    steps:
      - checkout
      - setup_google_dns
      - attach_workspace:
          at: ~/project
      - run:
          name: Load Docker Database Image
          command: |
            zstd -d litellm-docker-database.tar.zst --stdout | docker load
            docker images | grep litellm-docker-database
      - run:
          name: Install Dependencies
          command: |
            npm install -D @playwright/test
      - run:
          name: Install Playwright Browsers
          command: |
            npx playwright install
      - run:
          name: Run Docker container
          command: |
            docker run -d \
              -p 4000:4000 \
              -e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \
              -e LITELLM_MASTER_KEY="sk-1234" \
              -e OPENAI_API_KEY=$OPENAI_API_KEY \
              -e UI_USERNAME="admin" \
              -e UI_PASSWORD="gm" \
              -e LITELLM_LICENSE=$LITELLM_LICENSE \
              --name litellm-docker-database-<< parameters.browser >> \
              -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
              litellm-docker-database:ci \
              --config /app/config.yaml \
              --port 4000 \
              --detailed_debug
      - run:
          name: Install curl and dockerize
          command: |
            sudo apt-get update
            sudo apt-get install -y curl
            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start outputting logs
          command: docker logs -f litellm-docker-database-<< parameters.browser >>
          background: true
      - run:
          name: Wait for app to be ready
          command: dockerize -wait http://localhost:4000 -timeout 5m
      - run:
          name: Run Playwright Tests
          command: |
            npx playwright test \
              --project << parameters.browser >> \
              --config ui/litellm-dashboard/e2e_tests/playwright.config.ts \
              --reporter=html \
              --output=test-results
          no_output_timeout: 15m
      - store_artifacts:
          path: test-results
          destination: playwright-results

      - store_artifacts:
          path: playwright-report
          destination: playwright-report

  prisma_schema_sync:
    machine:
      image: ubuntu-2204:2023.10.1
    resource_class: medium
    working_directory: ~/project
    steps:
      - checkout
      - setup_google_dns
      - attach_workspace:
          at: ~/project
      - run:
          name: Install dockerize
          command: |
            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start PostgreSQL Database
          command: |
            docker run -d \
              --name postgres-db \
              -e POSTGRES_USER=postgres \
              -e POSTGRES_PASSWORD=postgres \
              -e POSTGRES_DB=litellm_schema_sync \
              -p 5432:5432 \
              postgres:14
      - run:
          name: Wait for PostgreSQL to be ready
          command: dockerize -wait tcp://localhost:5432 -timeout 1m
      - run:
          name: Load Docker Database Image
          command: |
            zstd -d litellm-docker-database.tar.zst --stdout | docker load
            docker images | grep litellm-docker-database
      - run:
          name: Run schema sync via prisma db push
          command: |
            docker run -d \
              -p 4000:4000 \
              -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_schema_sync" \
              -e LITELLM_MASTER_KEY="sk-1234" \
              --name schema-sync \
              --add-host=host.docker.internal:host-gateway \
              -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
              litellm-docker-database:ci \
              --config /app/config.yaml \
              --port 4000 \
              --use_prisma_db_push
      - run:
          name: Start outputting logs
          command: docker logs -f schema-sync
          background: true
      - run:
          name: Wait for proxy to be ready (schema sync complete)
          command: dockerize -wait http://localhost:4000 -timeout 5m
      - run:
          name: Stop schema sync container
          command: docker stop schema-sync


  test_bad_database_url:
    machine:
      image: ubuntu-2204:2023.10.1
    resource_class: medium
    working_directory: ~/project
    steps:
      - checkout
      - attach_workspace:
          at: ~/project
      - setup_google_dns
      - run:
          name: Install dockerize
          command: |
            wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
            rm dockerize-linux-amd64-v0.6.1.tar.gz
      - run:
          name: Start PostgreSQL Database
          command: |
            docker run -d \
              --name postgres-db \
              -e POSTGRES_USER=postgres \
              -e POSTGRES_PASSWORD=postgres \
              -e POSTGRES_DB=circle_test \
              -p 5432:5432 \
              postgres:14
      - run:
          name: Wait for PostgreSQL to be ready
          command: dockerize -wait tcp://localhost:5432 -timeout 1m
      - run:
          name: Load Docker Database Image
          command: |
            zstd -d litellm-docker-database.tar.zst --stdout | docker load
            docker tag litellm-docker-database:ci myapp:latest
      - run:
          name: Run Docker container with bad DATABASE_URL
          command: |
            docker run --name my-app \
              -p 4000:4000 \
              -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
              -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
              myapp:latest \
              --port 4000 > docker_output.log 2>&1 || true
      - run:
          name: Display Docker logs
          command: cat docker_output.log
      - run:
          name: Check for expected error
          command: |
            if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \
               (grep -q "Database setup failed after multiple retries" docker_output.log || \
                grep -q "ERROR:    Application startup failed. Exiting." docker_output.log); then
              echo "Expected error found. Test passed."
            else
              echo "Expected error not found. Test failed."
              cat docker_output.log
              exit 1
            fi

workflows:
  version: 2
  build_and_test:
    jobs:
      - using_litellm_on_windows:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - mypy_linting:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - semgrep:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - local_testing_part1:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - local_testing_part2:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - langfuse_logging_unit_tests:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
                - main
                - /litellm_.*/
                - main
                - /litellm_.*/
                - main
                - /litellm_.*/
                - main
                - /litellm_.*/
                - main
                - /litellm_.*/
      - litellm_assistants_api_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - litellm_router_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - litellm_router_unit_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - check_code_and_doc_quality:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - ui_build:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - ui_unit_tests:
          requires:
            - ui_build
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - auth_ui_unit_tests:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - build_docker_database_image:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - prisma_schema_sync:
          requires:
            - build_docker_database_image
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      # - e2e_ui_testing:
      #     name: e2e_ui_testing_chromium
      #     browser: chromium
      #     context: e2e_ui_tests
      #     requires:
      #       - ui_build
      #       - build_docker_database_image
      #       - prisma_schema_sync
      #     filters:
      #       branches:
      #         only:
      #           - main
      #           - /litellm_.*/
      # - e2e_ui_testing:
      #     name: e2e_ui_testing_firefox
      #     browser: firefox
      #     context: e2e_ui_tests
      #     requires:
      #       - ui_build
      #       - build_docker_database_image
      #       - prisma_schema_sync
      #     filters:
      #       branches:
      #         only:
      #           - main
      #           - /litellm_.*/
      - build_and_test:
          requires:
            - build_docker_database_image
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - e2e_openai_endpoints:
          requires:
            - build_docker_database_image
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - proxy_logging_guardrails_model_info_tests:
          requires:
            - build_docker_database_image
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - proxy_spend_accuracy_tests:
          requires:
            - build_docker_database_image
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - proxy_multi_instance_tests:
          requires:
            - build_docker_database_image
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - proxy_store_model_in_db_tests:
          requires:
            - build_docker_database_image
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - proxy_build_from_pip_tests:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - proxy_pass_through_endpoint_tests:
          requires:
            - build_docker_database_image
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - proxy_e2e_anthropic_messages_tests:
          requires:
            - build_docker_database_image
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - proxy_e2e_azure_batches_tests:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - llm_translation_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - realtime_translation_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - mcp_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - agent_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - guardrails_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - google_generate_content_endpoint_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - llm_responses_api_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - ocr_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - search_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - litellm_mapped_enterprise_tests:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - litellm_mapped_tests_proxy_part1:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - litellm_mapped_tests_proxy_part2:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
                - main
                - /litellm_.*/
                - main
                - /litellm_.*/
                - main
                - /litellm_.*/
                - main
                - /litellm_.*/
                - main
                - /litellm_.*/
      - batches_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - litellm_utils_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - pass_through_unit_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - image_gen_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - logging_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - audio_testing:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - upload-coverage:
          requires:
            - realtime_translation_testing
            - mcp_testing
            - agent_testing
            - google_generate_content_endpoint_testing
            - guardrails_testing
            - ocr_testing
            - search_testing
            - litellm_mapped_tests_proxy_part1
            - litellm_mapped_tests_proxy_part2
            - litellm_mapped_enterprise_tests
            - batches_testing
            - litellm_utils_testing
            - pass_through_unit_testing
            - image_gen_testing
            - logging_testing
            - audio_testing
            - langfuse_logging_unit_tests
            - local_testing_part1
            - local_testing_part2
            - litellm_assistants_api_testing
      - db_migration_disable_update_check:
          requires:
            - build_docker_database_image
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - installing_litellm_on_python:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - installing_litellm_on_python_3_13:
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - helm_chart_testing:
          requires:
            - build_docker_database_image
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - test_bad_database_url:
          requires:
            - build_docker_database_image
          filters:
            branches:
              only:
                - main
                - /litellm_.*/
      - publish_proxy_extras:
          filters:
            branches:
              only:
                - main
                - /litellm_release_day_.*/